def submit_workflow(workflow_name: Text = None) -> WorkflowInfo:
"""
Submits the user-defined workflow to the scheduler with the given name of workflow. Before the submission of
workflow in Scheduler Service, the metadata of workflow will be registered in Metadata Service.
The submission of workflow translates the current :class:`~ai_flow.ai_graph.ai_graph.AIGraph` , uploads the
project package, registers the metadata of the specified workflow and submits the workflow by Scheduler Service
which delegates the submission to the :class:`~ai_flow.plugin_interface.scheduler_interface.Scheduler`.
:param workflow_name: The name of the workflow.
:return: The :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo` which contains the information
about the submitted workflow.
"""
if current_graph().is_empty():
raise EmptyGraphException("Cannot submit empty graph")
entry_module_path = current_project_context().get_workflow_entry_module(workflow_name=workflow_name)
namespace = current_project_config().get_project_name()
translator = get_translator()
workflow = translator.translate(graph=current_graph(), project_context=current_project_context())
_apply_full_info_to_workflow(workflow, entry_module_path)
current_graph().clear_graph()
workflow_meta = get_ai_flow_client().get_workflow_by_name(project_name=current_project_config().get_project_name(),
workflow_name=workflow_name)
if workflow_meta is None:
get_ai_flow_client().register_workflow(name=workflow_name,
project_id=int(current_project_config().get_project_uuid()))
return proto_to_workflow(get_ai_flow_client()
.submit_workflow_to_scheduler(namespace=namespace,
workflow_json=json_utils.dumps(workflow),
workflow_name=workflow_name,
args={}))
def _upload_project_package(workflow: Workflow):
"""
Uploads the project package of the given :class:`~ai_flow.workflow.workflow.Workflow` by the
:class:`~ai_flow.plugin_interface.blob_manager_interface.BlobManager`.
:param workflow: The generated :class:`~ai_flow.workflow.workflow.Workflow`.
"""
blob_manager = BlobManagerFactory.get_blob_manager(current_project_config().get(WorkflowPropertyKeys.BLOB))
uploaded_project_path = blob_manager.upload_project(str(workflow.workflow_snapshot_id),
current_project_context().project_path)
workflow.project_uri = uploaded_project_path
workflow.properties[WorkflowPropertyKeys.BLOB] = current_project_config().get(WorkflowPropertyKeys.BLOB)
def list_workflow_executions(workflow_name: Text) -> List[WorkflowExecutionInfo]:
"""
Lists the :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowExecutionInfo` with the given workflow name.
:param workflow_name: The name of the workflow.
:return: List of the :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowExecutionInfo`.
"""
namespace = current_project_config().get_project_name()
return proto_to_workflow_execution_list(get_ai_flow_client().list_workflow_executions(namespace, workflow_name))
def delete_workflow(workflow_name: Text = None) -> WorkflowInfo:
"""
Deletes the workflow from the scheduler with the given name of workflow. Before the deletion of workflow in Scheduler
Service, the metadata of workflow will be deleted in Metadata Service.
The deletion of workflow deletes the metadata of the specified workflow, and removes the workflow by Scheduler
Service that assigns the deletion to the :class:`~ai_flow.plugin_interface.scheduler_interface.Scheduler`.
:param workflow_name: The name of the workflow.
:return: The :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo` which contains the information
about the deleted workflow.
"""
status: Status = get_ai_flow_client().delete_workflow_by_name(project_name=current_project_config()
.get_project_name(),
workflow_name=workflow_name)
if status == Status.ERROR:
raise Exception("Delete workflow {} failed")
return proto_to_workflow(get_ai_flow_client().delete_workflow(namespace=current_project_config().get_project_name(),
workflow_name=workflow_name))
def resume_workflow_scheduling(workflow_name: Text = None) -> WorkflowInfo:
"""
Resumes the workflow by the scheduler with the given name of workflow in Scheduler Service.
:param workflow_name: The name of the workflow.
:return: The :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo` which contains the information
about the scheduling resumed workflow.
"""
namespace = current_project_config().get_project_name()
return proto_to_workflow(get_ai_flow_client().resume_workflow_scheduling(namespace, workflow_name))
def test_init_ai_flow_context(self):
init_ai_flow_context()
project_config = current_project_config()
self.assertEqual('test_project', project_config.get_project_name())
self.assertEqual('a', project_config.get('a'))
project_context = current_project_context()
self.assertEqual('test_project', project_context.project_name)
workflow_config_ = current_workflow_config()
self.assertEqual('test_ai_flow_context', workflow_config_.workflow_name)
self.assertEqual(5, len(workflow_config_.job_configs))
def stop_all_workflow_executions(workflow_name: Text) -> List[WorkflowExecutionInfo]:
"""
Stops the workflow executions by the scheduler with the given name of workflow. The stop of the workflow execution
is delegated to the :class:`~ai_flow.plugin_interface.scheduler_interface.Scheduler` in Scheduler Service, that stops
the workflow running.
:param workflow_name: The name of the workflow.
:return: The :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowExecutionInfo` which contains the
information about the stopped workflow execution.
"""
namespace = current_project_config().get_project_name()
return proto_to_workflow_execution_list(get_ai_flow_client().kill_all_workflow_executions(namespace, workflow_name))
def start_new_workflow_execution(workflow_name: Text) -> WorkflowExecutionInfo:
"""
Starts the new workflow execution by the scheduler with the given name of workflow. The start of the workflow
execution is delegated to the :class:`~ai_flow.plugin_interface.scheduler_interface.Scheduler` in Scheduler Service,
that runs the workflow based on the current project path.
:param workflow_name: The name of the workflow.
:return: The :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowExecutionInfo` which contains the
information about the started workflow execution.
"""
namespace = current_project_config().get_project_name()
return proto_to_workflow_execution(get_ai_flow_client().start_new_workflow_execution(namespace, workflow_name))
def get_ai_flow_client():
""" Get AI flow Client. """
global _default_ai_flow_client, _default_server_uri
if _default_ai_flow_client is None:
current_uri = current_project_config().get_server_uri()
if current_uri is None:
return None
else:
_default_server_uri = current_uri
_default_ai_flow_client \
= AIFlowClient(server_uri=_default_server_uri,
notification_service_uri=current_project_config().get_notification_service_uri())
return _default_ai_flow_client
else:
current_uri = current_project_config().get_server_uri()
if current_uri != _default_server_uri:
_default_server_uri = current_uri
_default_ai_flow_client \
= AIFlowClient(server_uri=_default_server_uri,
notification_service_uri=current_project_config().get_notification_service_uri())
return _default_ai_flow_client
def test_init_job_runtime_context(self):
working_dir = os.path.dirname(__file__)
job_runtime_env = JobRuntimeEnv(
working_dir=working_dir,
job_execution_info=JobExecutionInfo(
job_name='task_1',
workflow_execution=WorkflowExecutionInfo(
workflow_execution_id='1',
workflow_info=WorkflowInfo(workflow_name='workflow_1'))))
init_job_runtime_context(job_runtime_env)
self.assertEqual('workflow_1', current_workflow_config().workflow_name)
self.assertEqual(
'task_1',
current_workflow_config().job_configs[current_job_name()].job_name)
self.assertEqual('test_project',
current_project_config().get_project_name())
def list_workflows(page_size: int, offset: int) -> Optional[List[WorkflowInfo]]:
"""
Lists the :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo` with the given pagination.
:param page_size: The limitation of the listed workflow pagination.
:param offset: The offset of the listed workflow pagination.
:return: Pagination list of the :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo`.
"""
project_name = current_project_config().get_project_name()
workflow_list = get_ai_flow_client().list_workflows(project_name=project_name,
page_size=page_size,
offset=offset)
if workflow_list is None:
return None
workflow_info_list = []
for workflow_meta in workflow_list:
workflow_info_list.append(WorkflowInfo(namespace=project_name, workflow_name=workflow_meta.name))
return workflow_info_list
def action_on_job_status(job_name: Text,
upstream_job_name: Text,
upstream_job_status: Status = Status.FINISHED,
action: JobAction = JobAction.START):
"""
Triggers the job with the given name when the status of the upstream job changes.
:param job_name: The name of the job triggered.
:param upstream_job_name: The name of the upstream job.
:param upstream_job_status: The status of the upstream job.
:param action: The :class:`~ai_flow.workflow.control_edge.JobAction` acts on the source channel whose value includes
START, RESTART, STOP and NONE.
"""
event_key = '.'.join(
[current_workflow_config().workflow_name, upstream_job_name])
action_on_event(job_name=job_name,
event_key=event_key,
event_type=AIFlowInternalEventType.JOB_STATUS_CHANGED,
sender=upstream_job_name,
event_value=upstream_job_status,
action=action,
namespace=current_project_config().get_project_name())
def __ensure_project_registered():
""" Ensure the project configured in project.yaml has been registered. """
client = get_ai_flow_client()
project_meta = client.get_project_by_name(
current_project_config().get_project_name())
pp = {}
for k, v in current_project_config().items():
pp[k] = str(v)
if project_meta is None:
project_meta = client.register_project(
name=current_project_config().get_project_name(), properties=pp)
else:
project_meta = client.update_project(
project_name=current_project_config().get_project_name(),
properties=pp)
current_project_config().set_project_uuid(str(project_meta.uuid))
def register_model(model_name: Text, model_desc: Text = None) -> ModelMeta:
project_config = current_project_config()
project_id = int(project_config.get_project_uuid())
return get_ai_flow_client().register_model(model_name, project_id, model_desc)
def get_workflow(workflow_name: Text = None) -> Optional[WorkflowInfo]:
"""
Gets the :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo` with the given name of workflow.
:param workflow_name: The name of the workflow.
:return: The :class:`~ai_flow.plugin_interface.scheduler_interface.WorkflowInfo` which contains the information
about the workflow.
"""
workflow_meta: WorkflowMeta = get_ai_flow_client().get_workflow_by_name(project_name=current_project_config()
.get_project_name(),
workflow_name=workflow_name)
if workflow_meta is None:
return None
project_meta: ProjectMeta = get_ai_flow_client().get_project_by_id(workflow_meta.project_id)
return WorkflowInfo(workflow_name=workflow_meta.name, namespace=project_meta.name)
def test_project_init_context(self):
project_path = os.path.dirname(__file__)
init_project_context(project_path)
self.assertEqual('test_project',
current_project_config().get_project_name())
self.assertEqual('a', current_project_config().get('a'))