def test_side_output_late_data(self):
self.env.set_parallelism(1)
config = Configuration(j_configuration=get_j_env_configuration(
self.env._j_stream_execution_environment))
config.set_integer('python.fn-execution.bundle.size', 1)
jvm = get_gateway().jvm
watermark_strategy = WatermarkStrategy(
jvm.org.apache.flink.api.common.eventtime.WatermarkStrategy.
forGenerator(jvm.org.apache.flink.streaming.api.functions.python.
eventtime.PerElementWatermarkGenerator.getSupplier())
).with_timestamp_assigner(SecondColumnTimestampAssigner())
tag = OutputTag('late-data',
type_info=Types.ROW([Types.STRING(),
Types.INT()]))
ds1 = self.env.from_collection(
[('a', 0), ('a', 8), ('a', 4), ('a', 6)],
type_info=Types.ROW([Types.STRING(), Types.INT()]))
ds2 = ds1.assign_timestamps_and_watermarks(watermark_strategy) \
.key_by(lambda e: e[0]) \
.window(TumblingEventTimeWindows.of(Time.milliseconds(5))) \
.allowed_lateness(0) \
.side_output_late_data(tag) \
.process(CountWindowProcessFunction(),
Types.TUPLE([Types.STRING(), Types.LONG(), Types.LONG(), Types.INT()]))
main_sink = DataStreamTestSinkFunction()
ds2.add_sink(main_sink)
side_sink = DataStreamTestSinkFunction()
ds2.get_side_output(tag).add_sink(side_sink)
self.env.execute('test_side_output_late_data')
main_expected = ['(a,0,5,1)', '(a,5,10,2)']
self.assert_equals_sorted(main_expected, main_sink.get_results())
side_expected = ['+I[a, 4]']
self.assert_equals_sorted(side_expected, side_sink.get_results())
def test_get_execution_environment_with_config(self):
configuration = Configuration()
configuration.set_integer('parallelism.default', 12)
configuration.set_string('pipeline.name', 'haha')
env = StreamExecutionEnvironment.get_execution_environment(configuration)
execution_config = env.get_config()
self.assertEqual(execution_config.get_parallelism(), 12)
config = Configuration(
j_configuration=get_j_env_configuration(env._j_stream_execution_environment))
self.assertEqual(config.get_string('pipeline.name', ''), 'haha')
def test_to_dict(self):
conf = Configuration()
conf.set_string("k1", "v1")
conf.set_integer("k2", 1)
conf.set_float("k3", 1.2)
conf.set_boolean("k4", True)
target_dict = conf.to_dict()
self.assertEqual(target_dict, {
"k1": "v1",
"k2": "1",
"k3": "1.2",
"k4": "true"
})
def test_remove_config(self):
conf = Configuration()
conf.set_string("k1", "v1")
conf.set_integer("k2", 1)
self.assertTrue(conf.contains_key("k1"))
self.assertTrue(conf.contains_key("k2"))
self.assertTrue(conf.remove_config("k1"))
self.assertFalse(conf.remove_config("k1"))
self.assertFalse(conf.contains_key("k1"))
conf.remove_config("k2")
self.assertFalse(conf.contains_key("k2"))
def test_hash_equal_str(self):
conf = Configuration()
conf2 = Configuration()
conf.set_string("k1", "v1")
conf.set_integer("k2", 1)
conf2.set_string("k1", "v1")
self.assertNotEqual(hash(conf), hash(conf2))
self.assertNotEqual(conf, conf2)
conf2.set_integer("k2", 1)
self.assertEqual(hash(conf), hash(conf2))
self.assertEqual(conf, conf2)
self.assertEqual(str(conf), "{k1=v1, k2=1}")
def test_add_all_to_dict(self):
conf = Configuration()
conf.set_string("k1", "v1")
conf.set_integer("k2", 1)
conf.set_float("k3", 1.2)
conf.set_boolean("k4", True)
conf.set_bytearray("k5", bytearray([1, 2, 3]))
target_dict = dict()
conf.add_all_to_dict(target_dict)
self.assertEqual(target_dict, {
"k1": "v1",
"k2": 1,
"k3": 1.2,
"k4": True,
"k5": bytearray([1, 2, 3])
})
def test_getters_and_setters(self):
conf = Configuration()
conf.set_string("str", "v1")
conf.set_integer("int", 2)
conf.set_boolean("bool", True)
conf.set_float("float", 0.5)
conf.set_bytearray("bytearray", bytearray([1, 2, 3]))
str_value = conf.get_string("str", "")
int_value = conf.get_integer("int", 0)
bool_value = conf.get_boolean("bool", False)
float_value = conf.get_float("float", 0)
bytearray_value = conf.get_bytearray("bytearray", bytearray())
self.assertEqual(str_value, "v1")
self.assertEqual(int_value, 2)
self.assertEqual(bool_value, True)
self.assertEqual(float_value, 0.5)
self.assertEqual(bytearray_value, bytearray([1, 2, 3]))