def test_side_output_late_data(self): self.env.set_parallelism(1) config = Configuration(j_configuration=get_j_env_configuration( self.env._j_stream_execution_environment)) config.set_integer('python.fn-execution.bundle.size', 1) jvm = get_gateway().jvm watermark_strategy = WatermarkStrategy( jvm.org.apache.flink.api.common.eventtime.WatermarkStrategy. forGenerator(jvm.org.apache.flink.streaming.api.functions.python. eventtime.PerElementWatermarkGenerator.getSupplier()) ).with_timestamp_assigner(SecondColumnTimestampAssigner()) tag = OutputTag('late-data', type_info=Types.ROW([Types.STRING(), Types.INT()])) ds1 = self.env.from_collection( [('a', 0), ('a', 8), ('a', 4), ('a', 6)], type_info=Types.ROW([Types.STRING(), Types.INT()])) ds2 = ds1.assign_timestamps_and_watermarks(watermark_strategy) \ .key_by(lambda e: e[0]) \ .window(TumblingEventTimeWindows.of(Time.milliseconds(5))) \ .allowed_lateness(0) \ .side_output_late_data(tag) \ .process(CountWindowProcessFunction(), Types.TUPLE([Types.STRING(), Types.LONG(), Types.LONG(), Types.INT()])) main_sink = DataStreamTestSinkFunction() ds2.add_sink(main_sink) side_sink = DataStreamTestSinkFunction() ds2.get_side_output(tag).add_sink(side_sink) self.env.execute('test_side_output_late_data') main_expected = ['(a,0,5,1)', '(a,5,10,2)'] self.assert_equals_sorted(main_expected, main_sink.get_results()) side_expected = ['+I[a, 4]'] self.assert_equals_sorted(side_expected, side_sink.get_results())
def test_get_execution_environment_with_config(self): configuration = Configuration() configuration.set_integer('parallelism.default', 12) configuration.set_string('pipeline.name', 'haha') env = StreamExecutionEnvironment.get_execution_environment(configuration) execution_config = env.get_config() self.assertEqual(execution_config.get_parallelism(), 12) config = Configuration( j_configuration=get_j_env_configuration(env._j_stream_execution_environment)) self.assertEqual(config.get_string('pipeline.name', ''), 'haha')
def test_to_dict(self): conf = Configuration() conf.set_string("k1", "v1") conf.set_integer("k2", 1) conf.set_float("k3", 1.2) conf.set_boolean("k4", True) target_dict = conf.to_dict() self.assertEqual(target_dict, { "k1": "v1", "k2": "1", "k3": "1.2", "k4": "true" })
def test_remove_config(self): conf = Configuration() conf.set_string("k1", "v1") conf.set_integer("k2", 1) self.assertTrue(conf.contains_key("k1")) self.assertTrue(conf.contains_key("k2")) self.assertTrue(conf.remove_config("k1")) self.assertFalse(conf.remove_config("k1")) self.assertFalse(conf.contains_key("k1")) conf.remove_config("k2") self.assertFalse(conf.contains_key("k2"))
def test_hash_equal_str(self): conf = Configuration() conf2 = Configuration() conf.set_string("k1", "v1") conf.set_integer("k2", 1) conf2.set_string("k1", "v1") self.assertNotEqual(hash(conf), hash(conf2)) self.assertNotEqual(conf, conf2) conf2.set_integer("k2", 1) self.assertEqual(hash(conf), hash(conf2)) self.assertEqual(conf, conf2) self.assertEqual(str(conf), "{k1=v1, k2=1}")
def test_add_all_to_dict(self): conf = Configuration() conf.set_string("k1", "v1") conf.set_integer("k2", 1) conf.set_float("k3", 1.2) conf.set_boolean("k4", True) conf.set_bytearray("k5", bytearray([1, 2, 3])) target_dict = dict() conf.add_all_to_dict(target_dict) self.assertEqual(target_dict, { "k1": "v1", "k2": 1, "k3": 1.2, "k4": True, "k5": bytearray([1, 2, 3]) })
def test_getters_and_setters(self): conf = Configuration() conf.set_string("str", "v1") conf.set_integer("int", 2) conf.set_boolean("bool", True) conf.set_float("float", 0.5) conf.set_bytearray("bytearray", bytearray([1, 2, 3])) str_value = conf.get_string("str", "") int_value = conf.get_integer("int", 0) bool_value = conf.get_boolean("bool", False) float_value = conf.get_float("float", 0) bytearray_value = conf.get_bytearray("bytearray", bytearray()) self.assertEqual(str_value, "v1") self.assertEqual(int_value, 2) self.assertEqual(bool_value, True) self.assertEqual(float_value, 0.5) self.assertEqual(bytearray_value, bytearray([1, 2, 3]))