def test_table_environment_with_blink_planner(self): t_env = BatchTableEnvironment.create( environment_settings=EnvironmentSettings.new_instance( ).in_batch_mode().use_blink_planner().build()) source_path = os.path.join(self.tempdir + '/streaming.csv') sink_path = os.path.join(self.tempdir + '/results') field_names = ["a", "b", "c"] field_types = [DataTypes.INT(), DataTypes.STRING(), DataTypes.STRING()] data = [(1, 'hi', 'hello'), (2, 'hello', 'hello')] csv_source = self.prepare_csv_source(source_path, data, field_types, field_names) t_env.register_table_source("source", csv_source) t_env.register_table_sink( "sink", CsvTableSink(field_names, field_types, sink_path)) source = t_env.scan("source") result = source.alias("a, b, c").select("1 + a, b, c") result.insert_into("sink") t_env.execute("blink_test") results = [] for root, dirs, files in os.walk(sink_path): for sub_file in files: with open(os.path.join(root, sub_file), 'r') as f: line = f.readline() while line is not None and line != '': results.append(line) line = f.readline() self.assert_equals(results, ['2,hi,hello\n', '3,hello,hello\n'])
def setUp(self): super(PyFlinkBatchTableTestCase, self).setUp() self.env = ExecutionEnvironment.get_execution_environment() self.env.set_parallelism(2) self.t_env = BatchTableEnvironment.create(self.env, TableConfig()) self.t_env.get_config().get_configuration().set_string( "taskmanager.memory.task.off-heap.size", "80mb")
def setUp(self): super(PyFlinkOldBatchTableTestCase, self).setUp() self.env = ExecutionEnvironment.get_execution_environment() self.env.set_parallelism(2) self.t_env = BatchTableEnvironment.create(self.env, TableConfig()) self.t_env.get_config().get_configuration().set_string( "python.fn-execution.bundle.size", "1")
def test_create_table_environment_with_old_planner(self): t_env = BatchTableEnvironment.create( environment_settings=EnvironmentSettings.new_instance().in_batch_mode() .use_old_planner().build()) self.assertEqual( t_env._j_tenv.getClass().getName(), "org.apache.flink.table.api.bridge.java.internal.BatchTableEnvironmentImpl")
def setUp(self): super(PyFlinkLegacyBlinkBatchTableTestCase, self).setUp() self.t_env = BatchTableEnvironment.create( environment_settings=EnvironmentSettings.new_instance() .in_batch_mode().use_blink_planner().build()) self.t_env._j_tenv.getPlanner().getExecEnv().setParallelism(2) self.t_env.get_config().get_configuration().set_string( "python.fn-execution.bundle.size", "1")
def setUp(self): super(PyFlinkBlinkBatchTableTestCase, self).setUp() self.t_env = BatchTableEnvironment.create( environment_settings=EnvironmentSettings.new_instance() .in_batch_mode().use_blink_planner().build()) self.t_env.get_config().get_configuration().set_string( "taskmanager.memory.task.off-heap.size", "80mb") self.t_env._j_tenv.getPlanner().getExecEnv().setParallelism(2)
def test_create_table_environment_with_blink_planner(self): t_env = BatchTableEnvironment.create( environment_settings=EnvironmentSettings.new_instance( ).in_batch_mode().use_blink_planner().build()) planner = t_env._j_tenv.getPlanner() self.assertEqual( planner.getClass().getName(), "org.apache.flink.table.planner.delegation.BatchPlanner")
def get_batch_table_environment(self) -> BatchTableEnvironment: """ Get the BatchTableEnvironment. If the BatchTableEnvironment has not been set, it initial the BatchTableEnvironment with default Configuration. :return: the BatchTableEnvironment. .. versionadded:: 1.11.0 """ if self._batch_tab_env is None: self._batch_tab_env = BatchTableEnvironment.create( ExecutionEnvironment.get_execution_environment()) return self._batch_tab_env
def test_create_table_environment(self): table_config = TableConfig() table_config.set_max_generated_code_length(32000) table_config.set_null_check(False) table_config.set_local_timezone("Asia/Shanghai") env = ExecutionEnvironment.get_execution_environment() t_env = BatchTableEnvironment.create(env, table_config) readed_table_config = t_env.get_config() self.assertFalse(readed_table_config.get_null_check()) self.assertEqual(readed_table_config.get_max_generated_code_length(), 32000) self.assertEqual(readed_table_config.get_local_timezone(), "Asia/Shanghai")
def test_explain_with_multi_sinks_with_blink_planner(self): t_env = BatchTableEnvironment.create( environment_settings=EnvironmentSettings.new_instance( ).in_batch_mode().use_blink_planner().build()) source = t_env.from_elements([(1, "Hi", "Hello"), (2, "Hello", "Hello")], ["a", "b", "c"]) field_names = ["a", "b", "c"] field_types = [ DataTypes.BIGINT(), DataTypes.STRING(), DataTypes.STRING() ] t_env.register_table_sink( "sink1", CsvTableSink(field_names, field_types, "path1")) t_env.register_table_sink( "sink2", CsvTableSink(field_names, field_types, "path2")) t_env.sql_update("insert into sink1 select * from %s where a > 100" % source) t_env.sql_update("insert into sink2 select * from %s where a < 100" % source) actual = t_env.explain(extended=True) self.assertIsInstance(actual, (str, unicode))
def setUp(self): super(PyFlinkBlinkBatchTableTestCase, self).setUp() self.t_env = BatchTableEnvironment.create( environment_settings=EnvironmentSettings.new_instance( ).in_batch_mode().use_blink_planner().build()) self.t_env._j_tenv.getPlanner().getExecEnv().setParallelism(2)
def setUp(self): super(PyFlinkBatchTableTestCase, self).setUp() self.env = ExecutionEnvironment.get_execution_environment() self.env.set_parallelism(2) self.t_env = BatchTableEnvironment.create(self.env, TableConfig())