def add_train_chief_alone_table(): stream_env = StreamExecutionEnvironment.get_execution_environment() table_env = StreamTableEnvironment.create(stream_env) statement_set = table_env.create_statement_set() work_num = 2 ps_num = 1 python_file = os.getcwd() + "/../../src/test/python/add.py" func = "map_func" prop = {} prop[TFCONSTANS.TF_IS_CHIEF_ALONE] = "true" prop[MLCONSTANTS.PYTHON_VERSION] = "3.7" env_path = None input_tb = None output_schema = None tf_config = TFConfig(work_num, ps_num, prop, python_file, func, env_path) train(stream_env, table_env, statement_set, input_tb, tf_config, output_schema) # inference(stream_env, table_env, statement_set, input_tb, tf_config, output_schema) job_client = statement_set.execute().get_job_client() if job_client is not None: job_client.get_job_execution_result( user_class_loader=None).result()
def input_output_table(): stream_env = StreamExecutionEnvironment.get_execution_environment() table_env = StreamTableEnvironment.create(stream_env) statement_set = table_env.create_statement_set() work_num = 2 ps_num = 1 python_file = os.getcwd() + "/../../src/test/python/input_output.py" prop = {} func = "map_func" env_path = None prop[ MLCONSTANTS. ENCODING_CLASS] = "org.flinkextended.flink.ml.operator.coding.RowCSVCoding" prop[ MLCONSTANTS. DECODING_CLASS] = "org.flinkextended.flink.ml.operator.coding.RowCSVCoding" inputSb = "INT_32" + "," + "INT_64" + "," + "FLOAT_32" + "," + "FLOAT_64" + "," + "STRING" prop["sys:csv_encode_types"] = inputSb prop["sys:csv_decode_types"] = inputSb prop[MLCONSTANTS.PYTHON_VERSION] = "3.7" source_file = os.getcwd() + "/../../src/test/resources/input.csv" sink_file = os.getcwd() + "/../../src/test/resources/output.csv" table_source = CsvTableSource(source_file, ["a", "b", "c", "d", "e"], [ DataTypes.INT(), DataTypes.BIGINT(), DataTypes.FLOAT(), DataTypes.DOUBLE(), DataTypes.STRING() ]) table_env.register_table_source("source", table_source) input_tb = table_env.from_path("source") output_schema = TableSchema(["a", "b", "c", "d", "e"], [ DataTypes.INT(), DataTypes.BIGINT(), DataTypes.FLOAT(), DataTypes.DOUBLE(), DataTypes.STRING() ]) sink = CsvTableSink(["a", "b", "c", "d", "e"], [ DataTypes.INT(), DataTypes.BIGINT(), DataTypes.FLOAT(), DataTypes.DOUBLE(), DataTypes.STRING() ], sink_file, write_mode=WriteMode.OVERWRITE) table_env.register_table_sink("table_row_sink", sink) tf_config = TFConfig(work_num, ps_num, prop, python_file, func, env_path) output_table = train(stream_env, table_env, statement_set, input_tb, tf_config, output_schema) # output_table = inference(stream_env, table_env, statement_set, input_tb, tf_config, output_schema) statement_set.add_insert("table_row_sink", output_table) job_client = statement_set.execute().get_job_client() if job_client is not None: job_client.get_job_execution_result( user_class_loader=None).result()
def worker_zero_finish(): stream_env = StreamExecutionEnvironment.get_execution_environment() table_env = StreamTableEnvironment.create(stream_env) work_num = 3 ps_num = 2 python_file = os.getcwd() + "/../../src/test/python/worker_0_finish.py" func = "map_func" prop = {MLCONSTANTS.PYTHON_VERSION: '3.7'} env_path = None input_tb = None output_schema = None tf_config = TFConfig(work_num, ps_num, prop, python_file, func, env_path) train(stream_env, table_env, input_tb, tf_config, output_schema) # inference(stream_env, table_env, input_tb, tf_config, output_schema) table_env.execute("train")
def fit(self, *inputs: Table) -> 'TensorflowModel': if len(inputs) == 0: if self.table_env is None: raise RuntimeError( "table_env should not be None if inputs is not given") input_table = None t_env = self.table_env else: input_table = inputs[0] t_env = input_table._t_env statement_set = self.statement_set if self.statement_set \ else t_env.create_statement_set() env = StreamExecutionEnvironment(t_env._j_tenv.execEnv()) train(env, t_env, statement_set, input_table, self.tf_config) return TensorflowModel(self.tf_config, statement_set, self.predict_col_names, self.predict_data_types)
def add_train_chief_alone_table(): stream_env = StreamExecutionEnvironment.get_execution_environment() table_env = StreamTableEnvironment.create(stream_env) work_num = 2 ps_num = 1 python_file = os.getcwd() + "/../../src/test/python/add.py" func = "map_func" prop = {} prop[TFCONSTANS.TF_IS_CHIEF_ALONE] = "true" prop[MLCONSTANTS.PYTHON_VERSION] = "3.7" env_path = None input_tb = None output_schema = None tf_config = TFConfig(work_num, ps_num, prop, python_file, func, env_path) train(stream_env, table_env, input_tb, tf_config, output_schema) # inference(stream_env, table_env, input_tb, tf_config, output_schema) table_env.execute("train")
def worker_zero_finish(): stream_env = StreamExecutionEnvironment.get_execution_environment() table_env = StreamTableEnvironment.create(stream_env) statement_set = table_env.create_statement_set() work_num = 3 ps_num = 2 python_file = os.getcwd() + "/../../src/test/python/worker_0_finish.py" func = "map_func" prop = {MLCONSTANTS.PYTHON_VERSION: '3.7'} env_path = None input_tb = None output_schema = None tf_config = TFConfig(work_num, ps_num, prop, python_file, func, env_path) train(stream_env, table_env, statement_set, input_tb, tf_config, output_schema) # inference(stream_env, table_env, statement_set, input_tb, tf_config, output_schema) job_client = statement_set.execute().get_job_client() if job_client is not None: job_client.get_job_execution_result(user_class_loader=None).result()
def input_output_table(): stream_env = StreamExecutionEnvironment.get_execution_environment() table_env = StreamTableEnvironment.create(stream_env) work_num = 2 ps_num = 1 python_file = os.getcwd() + "/../../src/test/python/input_output.py" prop = {} func = "map_func" env_path = None prop[MLCONSTANTS.ENCODING_CLASS] = "com.alibaba.flink.ml.operator.coding.RowCSVCoding" prop[MLCONSTANTS.DECODING_CLASS] = "com.alibaba.flink.ml.operator.coding.RowCSVCoding" inputSb = "INT_32" + "," + "INT_64" + "," + "FLOAT_32" + "," + "FLOAT_64" + "," + "STRING" prop["SYS:csv_encode_types"] = inputSb prop["SYS:csv_decode_types"] = inputSb prop[MLCONSTANTS.PYTHON_VERSION] = "3.7" source_file = os.getcwd() + "/../../src/test/resources/input.csv" table_source = CsvTableSource(source_file, ["a", "b", "c", "d", "e"], [DataTypes.INT(), DataTypes.INT(), DataTypes.FLOAT(), DataTypes.DOUBLE(), DataTypes.STRING()]) table_env.register_table_source("source", table_source) input_tb = table_env.scan("source") output_schema = TableSchema(["a", "b", "c", "d", "e"], [DataTypes.INT(), DataTypes.INT(), DataTypes.FLOAT(), DataTypes.DOUBLE(), DataTypes.STRING()] ) tf_config = TFConfig(work_num, ps_num, prop, python_file, func, env_path) train(stream_env, table_env, input_tb, tf_config, output_schema) # inference(stream_env, table_env, input_tb, tf_config, output_schema) table_env.execute("train")