def test_user_define_control_dependency(self):
print(sys._getframe().f_code.co_name)
trigger = af.external_trigger(name='stream_trigger')
job_config = af.BaseJobConfig('local', 'cmd_line')
job_config.job_name = 'test_cmd'
with af.config(job_config):
cmd_executor = af.user_define_operation(
output_num=0,
executor=CmdExecutor(
cmd_line="echo 'hello world' && sleep {}".format(1)))
af.user_define_control_dependency(
src=cmd_executor,
dependency=trigger,
event_key='key',
event_value='value',
event_type='name',
condition=MetCondition.NECESSARY,
action=TaskAction.START,
life=EventLife.ONCE,
value_condition=MetValueCondition.UPDATE)
workflow_id = af.submit_ai_flow()
af.get_ai_flow_client().publish_event('key', 'value1', 'name')
time.sleep(5)
af.get_ai_flow_client().publish_event('key', 'value2', 'name')
time.sleep(10)
af.stop_execution_by_id(workflow_id)
res = af.get_ai_flow_client().list_job(5, 0)
self.assertEqual(3, len(res))
def process(self, execution_context: ExecutionContext,
input_list: List) -> List:
model_meta: af.ModelMeta = execution_context.config.get('model_info')
model_name = model_meta.name
validated_model = af.get_latest_validated_model_version(model_name)
# Deprecate deployed model
deployed_model_version = af.get_deployed_model_version(model_name)
if deployed_model_version is not None:
af.update_model_version(
model_name=model_name,
model_version=deployed_model_version.version,
current_stage=ModelVersionStage.DEPRECATED)
af.update_model_version(model_name=model_name,
model_version=validated_model.version,
current_stage=ModelVersionStage.DEPLOYED)
af.get_ai_flow_client().send_event(
BaseEvent(key='START_PREDICTION', value=validated_model.version))
print(validated_model.version)
# Copy deployed model to deploy_model_dir
deployed_model_dir = af.get_artifact_by_name(self.artifact).uri
if not os.path.exists(deployed_model_dir):
os.makedirs(deployed_model_dir)
for file in os.listdir(deployed_model_dir):
file_path = os.path.join(deployed_model_dir, file)
if os.path.isfile(file_path):
os.remove(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path, True)
deployed_model_version = af.get_deployed_model_version(model_name)
shutil.copy(deployed_model_version.model_path, deployed_model_dir)
return []
def setup(self, execution_context: ExecutionContext):
# In this class, we show the usage of start_listen_event method which make it possible to send various events.
# Users can also refer `stream train stream predict` dataset to directly use provided API to get model version.
af.get_ai_flow_client().start_listen_event(key='START_PREDICTION',
watcher=self.watcher)
model_meta: af.ModelMeta = execution_context.config.get('model_info')
self.model_name = model_meta.name
print("### {} setup done for {}".format(self.__class__.__name__,
self.model_name))
def test_stream_with_external_trigger_with_model_control(self):
print(sys._getframe().f_code.co_name)
model_name = 'test_create_model_version'
model_desc = 'test create model version'
response = af.register_model(model_name=model_name,
model_type=af.ModelType.CHECKPOINT,
model_desc=model_desc)
trigger = af.external_trigger(name='stream_trigger')
job_config = af.BaseJobConfig('local', 'cmd_line')
job_config.job_name = 'test_cmd'
with af.config(job_config):
cmd_executor = af.user_define_operation(
output_num=0,
executor=CmdExecutor(
cmd_line="echo 'hello world' && sleep {}".format(1)))
af.model_version_control_dependency(
src=cmd_executor,
dependency=trigger,
model_name=model_name,
model_version_event_type='MODEL_DEPLOYED')
workflow_id = af.submit_ai_flow()
model_path1 = 'fs://source1.pkl'
model_metric1 = 'http://metric1'
model_flavor1 = '{"flavor.version":1}'
version_desc1 = 'test create model version1'
time.sleep(1)
response = af.register_model_version(
model=model_name,
model_path=model_path1,
model_metric=model_metric1,
model_flavor=model_flavor1,
version_desc=version_desc1,
current_stage=af.ModelVersionStage.DEPLOYED)
time.sleep(5)
response = af.register_model_version(
model=model_name,
model_path=model_path1,
model_metric=model_metric1,
model_flavor=model_flavor1,
version_desc=version_desc1,
current_stage=af.ModelVersionStage.DEPLOYED)
time.sleep(10)
af.stop_execution_by_id(workflow_id)
res = af.get_ai_flow_client().list_job(5, 0)
self.assertEqual(3, len(res))
def process(self, execution_context: ExecutionContext, input_list: List) -> List:
af.get_ai_flow_client().send_event(BaseEvent(key='k_2', value='v_2'))
return []