def _apply_full_info_to_workflow(workflow: Workflow, entry_module_path: Text):
"""
Applies the full information to the specified :class:`~ai_flow.workflow.workflow.Workflow` with the given entry
module path. The application of the workflow full information sets the entry module path, uploads the project
package of the workflow and set the registered job plugins.
:param workflow: The generated :class:`~ai_flow.workflow.workflow.Workflow`.
:param entry_module_path: The entry module path of the workflow.
"""
workflow.workflow_config = current_workflow_config()
_set_entry_module_path(workflow, entry_module_path)
_upload_project_package(workflow)
_set_job_plugins(workflow)
def build_workflow(self, split_graph: SplitGraph,
project_context: ProjectContext) -> Workflow:
workflow = Workflow()
workflow.workflow_config = current_workflow_config()
workflow.workflow_snapshot_id = '{}.{}.{}'.format(
project_context.project_name, workflow.workflow_name,
round(time.time() * 1000))
# add ai_nodes to workflow
for sub in split_graph.nodes.values():
if sub.config.job_type not in self.job_generator_registry.object_dict:
raise Exception("job generator not support job_type {}".format(
sub.config.job_type))
generator: JobGenerator = self.job_generator_registry \
.get_object(sub.config.job_type)
# set job resource dir
job_resource_dir = os.path.join(
project_context.get_generated_path(),
workflow.workflow_snapshot_id, sub.config.job_name)
if not os.path.exists(job_resource_dir):
os.makedirs(job_resource_dir)
job: Job = generator.generate(sub_graph=sub,
resource_dir=job_resource_dir)
job.resource_dir = job_resource_dir
# set input output dataset
for node in sub.nodes.values():
if isinstance(node, ReadDatasetNode):
job.input_dataset_list.append(node.dataset())
elif isinstance(node, WriteDatasetNode):
job.output_dataset_list.append(node.dataset())
workflow.add_job(job)
def validate_edge(head, tail):
if head not in workflow.jobs:
raise Exception(
'job: {} is not defined in workflow!'.format(head))
if tail is not None and tail != '' and tail != '*' and tail not in workflow.jobs:
raise Exception(
'job: {} is not defined in workflow!'.format(tail))
# add edges to workflow
for edges in split_graph.edges.values():
for e in edges:
control_edge = copy.deepcopy(e)
validate_edge(control_edge.destination, control_edge.source)
workflow.add_edge(control_edge.destination, control_edge)
return workflow
def test_workflow_serde(self):
workflow_config_file = os.path.join(os.path.dirname(__file__), 'workflow_1.yaml')
workflow_config = load_workflow_config(workflow_config_file)
workflow = Workflow()
workflow.workflow_config = workflow_config
jobs = []
for job_config in workflow_config.job_configs.values():
job = Job(job_config=job_config)
workflow.add_job(job)
jobs.append(job)
edge = ControlEdge(destination=jobs[0].job_name,
scheduling_rule=SchedulingRule(MeetAnyEventCondition().add_event('a', 'a'),
JobAction.START))
workflow.add_edge(jobs[0].job_name, edge)
edge = ControlEdge(destination=jobs[0].job_name,
scheduling_rule=SchedulingRule(MeetAnyEventCondition().add_event('b', 'b'),
JobAction.START))
workflow.add_edge(jobs[0].job_name, edge)
json_text = json_utils.dumps(workflow)
w: Workflow = json_utils.loads(json_text)
self.assertEqual(3, len(w.jobs))
self.assertEqual(2, len(w.edges.get(jobs[0].job_name)))
