def write_dataset(input: Channel,
dataset_info: Union[DatasetMeta, Text, int],
write_dataset_processor=None,
name: Text = None) -> None:
"""
Writes the dataset through the user-defined processor based on the metadata of dataset. The processor writes dataset
to external system and generates the :class:`~ai_flow.ai_graph.ai_node.WriteDatasetNode` with
:class:`~ai_flow.meta.dataset_meta.DatasetMeta`.
:param input: The input :class:`~ai_flow.graph.channel.Channel` from the operator that generates the data used to
write dataset.
:param dataset_info: Information about the dataset writen by the processor. The type of dataset information could be
:class:`~ai_flow.meta.dataset_meta.DatasetMeta` or Text(i.e. the name of dataset)
or int(i.e. the id of dataset). If the type is Text or int, this method will get
the metadata of dataset by the name or id of dataset by default.
:param write_dataset_processor: The user-defined processor composed of the logic that writes dataset.
:param name: Name of the writing dataset operator represent for the
:class:`~ai_flow.ai_graph.ai_node.WriteDatasetNode`.
:return: None.
"""
if isinstance(dataset_info, DatasetMeta):
dataset_meta = dataset_info
elif isinstance(dataset_info, Text):
dataset_meta = get_ai_flow_client().get_dataset_by_name(dataset_info)
else:
dataset_meta = get_ai_flow_client().get_dataset_by_id(dataset_info)
node = WriteDatasetNode(name=name,
processor=write_dataset_processor,
properties=None,
node_type='write_dataset',
dataset=dataset_meta)
add_ai_node_to_graph(node, inputs=input)
return None
def write_example(input_data: Channel,
example_info: Union[ExampleMeta, Text, int],
executor: Optional[PythonObjectExecutor] = None,
exec_args: ExecuteArgs = None) -> NoneChannel:
"""
Write example to example operator. It can write example to external system.
:param input_data: Channel from the specific operator which generates data.
:param example_info: Information about the example which will be read. Its type can be ExampleMeta
of py:class:`ai_flow.meta.example_meta.ExampleMeta` or Text or int. The example_info
means name in he metadata service when its type is Text and it means id when its type is int.
The ai flow will get the example from metadata service by name or id.
:param executor: The python user defined function in write example operator. User can write their own logic here.
:param exec_args: The properties of read example, there are batch properties, stream properties and
common properties respectively.
:return: NoneChannel.
"""
if isinstance(example_info, ExampleMeta):
example_meta = example_info
elif isinstance(example_info, Text):
example_meta = get_ai_flow_client().get_example_by_name(example_info)
else:
example_meta = get_ai_flow_client().get_example_by_id(example_info)
example_node = Example(example_meta=example_meta,
properties=exec_args,
is_source=False,
executor=executor)
_add_example_node_to_graph(example_node, input_data)
output: NoneChannel = example_node.outputs()[0]
return output
def read_dataset(dataset_info: Union[DatasetMeta, Text, int],
read_dataset_processor=None,
name: Text = None) -> Channel:
"""
Reads the dataset through the user-defined processor based on the metadata of dataset. The processor reads dataset
from external system and generates the :class:`~ai_flow.ai_graph.ai_node.ReadDatasetNode` with
:class:`~ai_flow.meta.dataset_meta.DatasetMeta`.
:param dataset_info: Information about the dataset read by the processor. The type of dataset information could be
:class:`~ai_flow.meta.dataset_meta.DatasetMeta` or Text(i.e. the name of dataset)
or int(i.e. the id of dataset). If the type is Text or int, this method will get
the metadata of dataset by the name or id of dataset by default.
:param read_dataset_processor: The user-defined processor composed of the logic that reads dataset.
:param name: Name of the reading dataset operator represent for the
:class:`~ai_flow.ai_graph.ai_node.ReadDatasetNode`.
:return: The :class:`~ai_flow.graph.channel.Channel`.
"""
if isinstance(dataset_info, DatasetMeta):
dataset_meta = dataset_info
elif isinstance(dataset_info, Text):
dataset_meta = get_ai_flow_client().get_dataset_by_name(dataset_info)
else:
dataset_meta = get_ai_flow_client().get_dataset_by_id(dataset_info)
node = ReadDatasetNode(name=name,
processor=read_dataset_processor,
properties=None,
node_type='read_dataset',
dataset=dataset_meta)
add_ai_node_to_graph(node, inputs=None)
outputs = node.outputs()
output: Channel = outputs[0]
return output
def submit_workflow(workflow_name: Text = None) -> WorkflowInfo:
"""
Submits the user-defined workflow to the scheduler with the given name of workflow. Before the submission of
workflow in Scheduler Service, the metadata of workflow will be registered in Metadata Service.
The submission of workflow translates the current :class:`~ai_flow.ai_graph.ai_graph.AIGraph` , uploads the
project package, registers the metadata of the specified workflow and submits the workflow by Scheduler Service
which delegates the submission to the :class:`~ai_flow.plugin_interface.scheduler_interface.Scheduler`.
:param workflow_name: The name of the workflow.
:return: The :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo` which contains the information
about the submitted workflow.
"""
if current_graph().is_empty():
raise EmptyGraphException("Cannot submit empty graph")
entry_module_path = current_project_context().get_workflow_entry_module(workflow_name=workflow_name)
namespace = current_project_config().get_project_name()
translator = get_translator()
workflow = translator.translate(graph=current_graph(), project_context=current_project_context())
_apply_full_info_to_workflow(workflow, entry_module_path)
current_graph().clear_graph()
workflow_meta = get_ai_flow_client().get_workflow_by_name(project_name=current_project_config().get_project_name(),
workflow_name=workflow_name)
if workflow_meta is None:
get_ai_flow_client().register_workflow(name=workflow_name,
project_id=int(current_project_config().get_project_uuid()))
return proto_to_workflow(get_ai_flow_client()
.submit_workflow_to_scheduler(namespace=namespace,
workflow_json=json_utils.dumps(workflow),
workflow_name=workflow_name,
args={}))
def register_model_version(model, model_path, model_type=None, version_desc=None,
current_stage=ModelVersionStage.GENERATED) -> ModelVersionMeta:
workflow_execution_id = None
if isinstance(model, str):
model_meta_info = get_ai_flow_client().get_model_by_name(model)
else:
model_meta_info = model
return get_ai_flow_client().register_model_version(model_meta_info, model_path, workflow_execution_id,
model_type, version_desc, current_stage)
def train(input: Union[Channel, List[Channel]],
model_info: Union[ModelMeta, Text, int],
training_processor,
base_model_info: Union[ModelMeta, Text, int] = None,
output_num=0,
name: Text = None) -> Union[Channel, Tuple[Channel]]:
"""
Trains model through the user-defined processor with the information of models based on the input
:class:`~ai_flow.graph.channel.Channel`. The processor generates the model training
:class:`~ai_flow.ai_graph.ai_node.AINode`.
:param input: List of the input :class:`~ai_flow.graph.channel.Channel` that contains the multiple channels from the
operators which generate the data used to train model.
:param model_info: Information about the model trained by the processor. The type of model could be
:class:`~ai_flow.meta.model_meta.ModelMeta` or Text(i.e. the name of model) or int(
i.e. the id of model). If the type is Text or int, this method will get the
metadata of model by the name or id of model by default.
:param training_processor: The user-defined processor composed of the logic that trains model.
:param base_model_info: Information about the base model for training by the processor. The type of model could be
:class:`~ai_flow.meta.model_meta.ModelMeta` or Text or int. The
`base_model_info` means the name of model when the type is Text and it
means the id of model when the type is int, which gets the metadata of
model by the id or name of model.
:param output_num: The output :class:`~ai_flow.graph.channel.Channel` number of the model training operator.
The default value is 0.
:param name: Name of the model training operator represent for the :class:`~ai_flow.ai_graph.ai_node.AINode`.
:return: Returns the :class:`~ai_flow.graph.channel.Channel` when the `output_num` is 1 and returns
Tuple[:class:`~ai_flow.graph.channel.Channel`] when the `output_num` is bigger than 1.
"""
if isinstance(model_info, ModelMeta):
output_model_meta = model_info
elif isinstance(model_info, Text):
output_model_meta = get_ai_flow_client().get_model_by_name(model_info)
else:
output_model_meta = get_ai_flow_client().get_model_by_id(model_info)
if base_model_info is not None:
if isinstance(base_model_info, ModelMeta):
base_model_meta = base_model_info
elif isinstance(base_model_info, Text):
base_model_meta = get_ai_flow_client().get_model_by_name(
base_model_info)
else:
base_model_meta = get_ai_flow_client().get_model_by_id(
base_model_info)
else:
base_model_meta = None
return user_define_operation(input=input,
name=name,
processor=training_processor,
model_info=output_model_meta,
base_model_info=base_model_meta,
output_num=output_num,
operation_type='train')
def start_listen_event(key: str, watcher: EventWatcher, version=None):
"""
Start listen specific `key` or `version` events in Notification Service.
:param key: Key of event for listening.
:param watcher: Watcher instance for listening notification.
:param version: (Optional) Version of event for listening.
"""
get_ai_flow_client().start_listen_event(key=key,
watcher=watcher,
version=version)
def predict(input: Union[Channel, List[Channel]],
model_info: Union[ModelMeta, Text, int],
prediction_processor,
model_version_info: Optional[Union[ModelVersionMeta, Text]] = None,
output_num=1,
name: Text = None) -> Union[Channel, Tuple[Channel]]:
"""
Predicts through the user-defined processor with the information of model and model version based on the input
:class:`~ai_flow.graph.channel.Channel`. The processor generates the model prediction
:class:`~ai_flow.ai_graph.ai_node.AINode`.
:param input: List of the input :class:`~ai_flow.graph.channel.Channel` that contains the multiple channels from the
operators which generate the data used to predict model.
:param model_info: Information about the model predicted by the processor. The type of model could be
:class:`~ai_flow.meta.model_meta.ModelMeta` or Text(i.e. the name of model) or int(
i.e. the id of model). If the type is Text or int, this method will get the
metadata of model by the name or id of model by default.
:param prediction_processor: The user-defined processor composed of the logic that predicts model.
:param model_version_info: Information about the model version predicted by the processor. The type of model version
could be :class:`~ai_flow.meta.model_meta.ModelVersionMeta` or Text(
i.e. the `version` field of
:class:`~ai_flow.meta.model_meta.ModelVersionMeta`). If the type is
Text, this method will get the metadata of model version by the
version of model version by default.
:param output_num: The output :class:`~ai_flow.graph.channel.Channel` number of the model prediction operator.
The default value is 1.
:param name: Name of the model prediction operator represent for the :class:`~ai_flow.ai_graph.ai_node.AINode`.
:return: Returns the :class:`~ai_flow.graph.channel.Channel` when the `output_num` is 1 and returns
Tuple[:class:`~ai_flow.graph.channel.Channel`] when the `output_num` is bigger than 1.
"""
if isinstance(model_info, ModelMeta):
model_meta = model_info
elif isinstance(model_info, Text):
model_meta = get_ai_flow_client().get_model_by_name(model_info)
else:
model_meta = get_ai_flow_client().get_model_by_id(model_info)
if isinstance(model_version_info, ModelVersionMeta):
model_version_meta = model_version_info
elif isinstance(model_version_info, Text):
model_version_meta = get_ai_flow_client() \
.get_model_version_by_version(model_version_info,
get_ai_flow_client().get_model_by_name(model_meta.name).uuid)
else:
model_version_meta = None
return user_define_operation(input=input,
name=name,
model_info=model_meta,
processor=prediction_processor,
model_version_info=model_version_meta,
output_num=output_num,
operation_type='predict')
def evaluate(input_data_list: List[Channel],
model_info: Union[ModelMeta, Text, int],
executor: BaseExecutor,
exec_args: ExecuteArgs = None,
output_num=0,
name: Text = None) -> Union[NoneChannel, Channel, Tuple[Channel]]:
"""
Evaluate Operator. Do evaluate job with the specific model version.
:param input_data_list: List of Channel. It contains the example data used in prediction.
:param model_info: Information about the model which is in prediction. Its type can be ModelMeta
of py:class:`ai_flow.meta.model_meta.ModelMeta` or Text or int. The model_info
means name in he metadata service when its type is Text and it means id when its type is
int. The ai flow will get the model meta from metadata service by name or id.
:param executor: The user defined function in evaluate operator. User can write their own logic here.
:param exec_args: The properties of read example, there are batch properties, stream properties and
common properties respectively.
:param output_num: The output number of the operator. The default value is 0.
:param name: Name of the predict operator.
:return: NoneChannel.
"""
if isinstance(model_info, ModelMeta):
model_meta = model_info
elif isinstance(model_info, Text):
model_meta = get_ai_flow_client().get_model_by_name(model_info)
else:
model_meta = get_ai_flow_client().get_model_by_id(model_info)
node = Evaluator(name=name,
model=model_meta,
executor=executor,
properties=exec_args,
output_num=output_num)
_add_execute_node_to_graph(executor=executor,
node=node,
inputs=input_data_list)
outputs = node.outputs()
if isinstance(outputs[0], NoneChannel):
output: NoneChannel = outputs[0]
return output
elif 1 == len(outputs):
output: Channel = outputs[0]
return output
else:
output: List[Channel] = []
for i in outputs:
tmp: Channel = i
output.append(tmp)
return tuple(output)
def get_workflow(workflow_name: Text = None) -> Optional[WorkflowInfo]:
"""
Gets the :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo` with the given name of workflow.
:param workflow_name: The name of the workflow.
:return: The :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo` which contains the information
about the workflow.
"""
workflow_meta: WorkflowMeta = get_ai_flow_client().get_workflow_by_name(project_name=current_project_config()
.get_project_name(),
workflow_name=workflow_name)
if workflow_meta is None:
return None
project_meta: ProjectMeta = get_ai_flow_client().get_project_by_id(workflow_meta.project_id)
return WorkflowInfo(workflow_name=workflow_meta.name, namespace=project_meta.name)
def push_model(model_info: Union[ModelMeta, Text, int],
executor: BaseExecutor,
model_version_info: Optional[Union[ModelVersionMeta,
Text]] = None,
exec_args: ExecuteArgs = None,
name: Text = None) -> NoneChannel:
"""
Pusher operator. The pusher operator is used to push a validated model which is better than previous one to
a deployment target.
:param model_info: Information about the model which is in pusher. Its type can be ModelMeta
of py:class:`ai_flow.meta.model_meta.ModelMeta` or Text or int. The model_info
means name in he metadata service when its type is Text and it means id when its type is
int. The ai flow will get the model meta from metadata service by name or id.
:param executor: The user defined function in pusher operator. User can write their own logic here.
:param model_version_info: Information about the model version which is in push. Its type can be
ModelVersionMeta of py:class:`ai_flow.meta.model_meta.ModelVersionMeta`
or Text. The model_version_info means version in he metadata service
when its type is Text. The ai flow will get the model meta from metadata
service by version.
:param exec_args: The properties of read example, there are batch properties, stream properties and
common properties respectively.
:param name: Name of the push operator.
:return: NoneChannel.
"""
if isinstance(model_info, ModelMeta) or model_info is None:
model_meta = model_info
elif isinstance(model_info, Text):
model_meta = get_ai_flow_client().get_model_by_name(model_info)
else:
model_meta = get_ai_flow_client().get_model_by_id(model_info)
if isinstance(model_version_info,
ModelVersionMeta) or model_version_info is None:
model_version_info = None
elif isinstance(model_version_info, Text):
model_version_info = get_ai_flow_client().get_model_version_by_version(
version=model_version_info, model_id=model_meta.uuid)
node = Pusher(model=model_meta,
executor=executor,
properties=exec_args,
model_version=model_version_info,
name=name)
_add_execute_node_to_graph(executor, node, inputs=[])
outputs = node.outputs()
output: NoneChannel = outputs[0]
return output
def submit_workflow(workflow_name: Text = None,
args: Dict = None) -> WorkflowInfo:
"""
Submit the ai flow workflow to the scheduler.
:param workflow_name: The ai flow workflow identify.
:param args: The arguments of the submit action.
:return: The result of the submit action.
"""
call_path = os.path.abspath(sys._getframe(1).f_code.co_filename)
project_path = os.path.abspath(project_description().project_path)
# length /python_codes/ is 14; length .py is 3
entry_module_path = call_path[len(project_path) + 14:-3].replace('/', '.')
namespace = project_config().get_project_name()
translator = get_default_translator()
workflow = translator.translate(graph=default_graph(),
project_desc=project_description())
for job in workflow.jobs.values():
_register_job_meta(workflow_id=workflow.workflow_id, job=job)
_set_entry_module_path(workflow, entry_module_path)
_upload_project_package(workflow)
return proto_to_workflow(get_ai_flow_client().submit_workflow_to_scheduler(
namespace=namespace,
workflow_json=json_utils.dumps(workflow),
workflow_name=workflow_name,
args=args))
def list_jobs(execution_id: Text) -> List[JobInfo]:
"""
List the jobs of the workflow execution.
:param execution_id:
:return:
"""
return proto_to_job_list(get_ai_flow_client().list_jobs(execution_id))
def list_workflows() -> List[WorkflowInfo]:
"""
:return: All workflow information.
"""
namespace = project_config().get_project_name()
return proto_to_workflow_list(
get_ai_flow_client().list_workflows(namespace))
def get_workflow_execution(execution_id: Text) -> WorkflowExecutionInfo:
"""
Get the WorkflowExecutionInfo from scheduler.
:param execution_id:
:return: WorkflowExecutionInfo
"""
return proto_to_workflow_execution(
get_ai_flow_client().get_workflow_execution(execution_id))
def register_project(name,
uri: Text = None,
properties: Properties = None,
user: Text = None,
password: Text = None,
project_type: Text = None) -> ProjectMeta:
return get_ai_flow_client().register_project(name, uri, properties, user,
password, project_type)
def update_metric_meta(uuid: int, name: Text = None, dataset_id: int = None, model_name: Optional[Text] = None,
model_version: Optional[Text] = None, job_id: int = None, start_time: int = None,
end_time: int = None, metric_type: MetricType = MetricType.DATASET, uri: Text = None,
tags: Text = None, metric_description: Text = None, properties: Properties = None) \
-> Tuple[int, Text, Optional[MetricMeta]]:
return get_ai_flow_client().update_metric_meta(
uuid, name, dataset_id, model_name, model_version, job_id, start_time,
end_time, metric_type, uri, tags, metric_description, properties)
def stop_job(job_name: Text, execution_id: Text) -> JobInfo:
"""
Stop a job defined in the ai flow workflow.
:param job_name: The job name which task defined in workflow.
:param execution_id: The ai flow workflow execution identify.
:return: The result of the action.
"""
return proto_to_job(get_ai_flow_client().stop_job(job_name, execution_id))
def get_job(job_name: Text, execution_id: Text) -> JobInfo:
"""
Get job information by job name.
:param job_name:
:param execution_id:
:return:
"""
return proto_to_job(get_ai_flow_client().get_job(job_name, execution_id))
def list_events(key: str, version: int = None) -> list:
"""
List specific `key` or `version` events in Notification Service.
:param key: Key of event for listening.
:param version: (Optional) Version of event for listening.
"""
return get_ai_flow_client().list_events(key=key, version=version)
def kill_workflow_execution(execution_id: Text) -> WorkflowExecutionInfo:
"""
Stop the instance of the workflow.
:param execution_id: The ai flow workflow execution identify.
:return: The result of the action.
"""
return proto_to_workflow_execution(
get_ai_flow_client().kill_workflow_execution(execution_id))
def list_job_executions(execution_id: Text) -> List[JobExecutionInfo]:
"""
Lists the :class:`~ai_flow.plugin_interface.scheduler_interface.JobExecutionInfo` with the given id of workflow
execution.
:param execution_id: The id of the workflow execution.
:return: List of the :class:`~ai_flow.plugin_interface.scheduler_interface.JobExecutionInfo`.
"""
return proto_to_job_list(get_ai_flow_client().list_jobs(execution_id))
def list_workflow_executions(workflow_name: Text) -> List[WorkflowExecutionInfo]:
"""
Lists the :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowExecutionInfo` with the given workflow name.
:param workflow_name: The name of the workflow.
:return: List of the :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowExecutionInfo`.
"""
namespace = current_project_config().get_project_name()
return proto_to_workflow_execution_list(get_ai_flow_client().list_workflow_executions(namespace, workflow_name))
def update_project(project_name: Text,
uri: Text = None,
properties: Properties = None,
user: Text = None,
password: Text = None,
project_type: Text = None) -> Optional[ProjectMeta]:
return get_ai_flow_client().update_project(project_name, uri, properties,
project_type, user, password,
project_type)
def get_workflow(workflow_name: Text = None) -> WorkflowInfo:
"""
Return the workflow information.
:param workflow_name: The ai flow workflow identify.
:return: the workflow information.
"""
namespace = project_config().get_project_name()
return proto_to_workflow(get_ai_flow_client().get_workflow(
namespace, workflow_name))
def register_dataset_with_catalog(name: Text,
catalog_name: Text, catalog_type: Text,
catalog_connection_uri: Text,
catalog_table: Text, catalog_database: Text = None) -> DatasetMeta:
return get_ai_flow_client().register_dataset_with_catalog(name=name,
catalog_name=catalog_name, catalog_type=catalog_type,
catalog_connection_uri=catalog_connection_uri,
catalog_table=catalog_table,
catalog_database=catalog_database)
def resume_workflow_scheduling(workflow_name: Text = None) -> WorkflowInfo:
"""
Resume the ai flow workflow from the scheduler.
:param workflow_name: The ai flow workflow identify.
:return: The result of the action.
"""
namespace = project_config().get_project_name()
return proto_to_workflow(get_ai_flow_client().resume_workflow_scheduling(
namespace, workflow_name))
def register_metric_summary(metric_name: Text,
metric_key: Text,
metric_value: Text,
metric_timestamp: int,
model_version: Optional[Text] = None,
job_execution_id: Optional[Text] = None
) -> Tuple[int, Text, Optional[MetricSummary]]:
return get_ai_flow_client().register_metric_summary(metric_name, metric_key, metric_value, metric_timestamp,
model_version, job_execution_id)
def resume_workflow_scheduling(workflow_name: Text = None) -> WorkflowInfo:
"""
Resumes the workflow by the scheduler with the given name of workflow in Scheduler Service.
:param workflow_name: The name of the workflow.
:return: The :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo` which contains the information
about the scheduling resumed workflow.
"""
namespace = current_project_config().get_project_name()
return proto_to_workflow(get_ai_flow_client().resume_workflow_scheduling(namespace, workflow_name))
def get_workflow_execution(execution_id: Text) -> WorkflowExecutionInfo:
"""
Gets the :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowExecutionInfo` with the given id of workflow
execution.
:param execution_id: The id of the workflow execution.
:return: The :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowExecutionInfo` which contains the
information about the workflow execution.
"""
return proto_to_workflow_execution(get_ai_flow_client().get_workflow_execution(execution_id))
