def test_stream_train_component(self):
batch_input_example_meta = af.register_example(
name='stream_train_example',
support_type=ExampleSupportType.EXAMPLE_BOTH)
model_meta = af.register_model(model_name='mnist_model',
model_type=ModelType.SAVED_MODEL)
stream_input_example_meta = af.register_example(
name='stream_train_example',
support_type=ExampleSupportType.EXAMPLE_BOTH)
with af.config(
af.BaseJobConfig(platform='local',
engine='python',
job_name='stream_train')):
batch_input_example = af.read_example(
example_info=batch_input_example_meta,
executor=PythonObjectExecutor(
python_object=ReadBatchExample()))
batch_train = af.train(input_data_list=[batch_input_example],
executor=PythonObjectExecutor(
python_object=TrainBatchMnistModel()),
model_info=model_meta)
stream_input_example = af.read_example(
example_info=stream_input_example_meta,
executor=PythonObjectExecutor(
python_object=ReadStreamExample()))
stream_train = af.train(input_data_list=[stream_input_example],
executor=PythonObjectExecutor(
python_object=TrainStreamMnistModel()),
model_info=model_meta)
af.stop_before_control_dependency(stream_train, batch_train)
workflow_id = af.run(test_util.get_project_path())
res = af.wait_workflow_execution_finished(workflow_id)
self.assertEqual(0, res)
def test_batch_train_component_with_an_output(self):
input_example_meta = af.register_example(
name='batch_train_example',
support_type=ExampleSupportType.EXAMPLE_BATCH)
model_meta = af.register_model(model_name='mnist_model',
model_type=ModelType.SAVED_MODEL)
example_meta = af.register_example(
name='output_example',
support_type=ExampleSupportType.EXAMPLE_BATCH,
data_type='numpy',
data_format='npz',
batch_uri=os.path.abspath(
os.path.dirname(__file__) + '/numpy_output.npz'))
with af.config(
af.BaseJobConfig(platform='local',
engine='python',
job_name='batch_train')):
input_example = af.read_example(
example_info=input_example_meta,
executor=PythonObjectExecutor(
python_object=ReadBatchExample()))
train_channel = af.train(
input_data_list=[input_example],
executor=PythonObjectExecutor(
python_object=TrainBatchMnistModelWithOutput()),
model_info=model_meta,
output_num=1)
af.write_example(input_data=train_channel,
example_info=example_meta)
workflow_id = af.run(test_util.get_project_path())
res = af.wait_workflow_execution_finished(workflow_id)
self.assertEqual(0, res)
def test_stream_transform_component(self):
file = get_file_dir(__file__) + '/test1.csv'
input_example_meta = af.register_example(
name='test_example',
support_type=ExampleSupportType.EXAMPLE_BOTH,
stream_uri=file)
output_file = get_file_dir(
__file__) + "/output_transform_stream_test1.csv"
output_example_meta = af.register_example(
name='test_example_output',
support_type=ExampleSupportType.EXAMPLE_BOTH,
stream_uri=output_file)
with af.config(
af.BaseJobConfig(platform='local',
engine='python',
job_name='stream_transform')):
input_example = af.read_example(
example_info=input_example_meta,
executor=PythonObjectExecutor(
python_object=ReadStreamExample()))
transform_example = af.transform(
input_data_list=[input_example],
executor=PythonObjectExecutor(
python_object=TransformStreamData()))
af.write_example(input_data=transform_example,
example_info=output_example_meta.name,
executor=PythonObjectExecutor(
python_object=WriteStreamExample()))
workflow_id = af.run(test_util.get_project_path())
res = af.wait_workflow_execution_finished(workflow_id)
self.assertEqual(0, res)
def test_stream_evaluate_component(self):
input_example_meta = af.register_example(
name='batch_train_example',
support_type=ExampleSupportType.EXAMPLE_BATCH)
model_meta = af.register_model(model_name='mnist_model',
model_type=ModelType.SAVED_MODEL)
stream_evaluate_example_meta = af.register_example(
name='stream_evaluate_example',
support_type=ExampleSupportType.EXAMPLE_STREAM)
stream_output_file = get_file_dir(__file__) + '/stream_evaluate'
evaluate_output = af.register_artifact(name='stream_evaluate',
stream_uri=stream_output_file)
stream_evaluate_result_example_meta = af.register_example(
name='stream_evaluate_result_example',
support_type=ExampleSupportType.EXAMPLE_STREAM,
stream_uri=stream_output_file)
if os.path.exists(stream_output_file):
os.remove(stream_output_file)
with af.config(
af.BaseJobConfig(platform='local',
engine='python',
job_name='stream_evaluate')):
input_example = af.read_example(
example_info=input_example_meta,
executor=PythonObjectExecutor(
python_object=ReadBatchExample()))
batch_train = af.train(input_data_list=[input_example],
executor=PythonObjectExecutor(
python_object=TrainBatchMnistModel()),
model_info=model_meta)
stream_evaluate_example = af.read_example(
example_info=stream_evaluate_example_meta,
executor=PythonObjectExecutor(
python_object=ReadStreamExample()))
stream_evaluate = af.evaluate(
input_data_list=[stream_evaluate_example],
model_info=model_meta,
executor=PythonObjectExecutor(
python_object=EvaluateStreamMnistModel()),
output_num=1)
af.write_example(input_data=stream_evaluate,
example_info=stream_evaluate_result_example_meta,
executor=PythonObjectExecutor(
python_object=WriteStreamExample()))
af.stop_before_control_dependency(stream_evaluate, batch_train)
workflow_id = af.run(test_util.get_project_path())
res = af.wait_workflow_execution_finished(workflow_id)
self.assertEqual(0, res)
def test_batch_predict_component(self):
input_example_meta = af.register_example(
name='input_train_example',
support_type=ExampleSupportType.EXAMPLE_BOTH)
model_meta = af.register_model(model_name='mnist_model',
model_type=ModelType.SAVED_MODEL)
batch_output_file = get_file_dir(__file__) + '/batch_predict'
evaluate_output = af.register_artifact(name='batch_evaluate',
batch_uri=batch_output_file)
output_example_meta = af.register_example(
name='output_result_example',
support_type=ExampleSupportType.EXAMPLE_BATCH,
data_type='numpy',
data_format='txt',
batch_uri=batch_output_file)
if os.path.exists(batch_output_file):
os.remove(batch_output_file)
with af.config(
af.BaseJobConfig(platform='local',
engine='python',
job_name='batch_predict')):
batch_example = af.read_example(
example_info=input_example_meta,
executor=PythonObjectExecutor(
python_object=ReadBatchExample()))
batch_train = af.train(input_data_list=[batch_example],
executor=PythonObjectExecutor(
python_object=TrainBatchMnistModel()),
model_info=model_meta)
batch_predict = af.predict(
input_data_list=[batch_example],
model_info=model_meta,
executor=PythonObjectExecutor(
python_object=PredictBatchMnistModel()),
output_num=1)
af.write_example(input_data=batch_predict,
example_info=output_example_meta)
af.stop_before_control_dependency(batch_predict, batch_train)
workflow_id = af.run(test_util.get_project_path())
res = af.wait_workflow_execution_finished(workflow_id)
self.assertEqual(0, res)
def test_batch_train_component(self):
input_example_meta = af.register_example(
name='batch_train_example',
support_type=ExampleSupportType.EXAMPLE_BATCH)
model_meta = af.register_model(model_name='mnist_model',
model_type=ModelType.SAVED_MODEL)
with af.config(
af.BaseJobConfig(platform='local',
engine='python',
job_name='batch_train')):
input_example = af.read_example(
example_info=input_example_meta,
executor=PythonObjectExecutor(
python_object=ReadBatchExample()))
af.train(input_data_list=[input_example],
executor=PythonObjectExecutor(
python_object=TrainBatchMnistModel()),
model_info=model_meta)
workflow_id = af.run(test_util.get_project_path())
res = af.wait_workflow_execution_finished(workflow_id)
self.assertEqual(0, res)