def deploy_to_airflow(project_path: Text = None,
dag_id: Text = None,
default_args=None):
"""
Run project under the current project path.
:param project_path: The description of the project..
:param dag_id: The airflow dag id.
:param default_args:
:return: (airflow dag file path, airflow dag code).
"""
project_desc = generate_project_desc(project_path)
if dag_id is None:
dag_id = default_af_job_context().global_workflow_config.name
if dag_id is None:
dag_id = project_desc.project_name
deploy_path = project_desc.project_config.get_airflow_deploy_path()
if deploy_path is None:
raise Exception("airflow_deploy_path config not set!")
airflow_file_path = deploy_path + '/' + dag_id + '.py'
if os.path.exists(airflow_file_path):
os.remove(airflow_file_path)
generated_code = _generate_airflow_file_text(ai_graph=default_graph(),
project_desc=project_desc,
dag_id=dag_id,
default_args=default_args)
with NamedTemporaryFile(mode='w+t',
prefix=dag_id,
suffix='.py',
dir='/tmp',
delete=False) as f:
f.write(generated_code)
os.rename(f.name, airflow_file_path)
return airflow_file_path, generated_code
def submit_workflow(workflow_name: Text = None,
args: Dict = None) -> WorkflowInfo:
"""
Submit the ai flow workflow to the scheduler.
:param workflow_name: The ai flow workflow identify.
:param args: The arguments of the submit action.
:return: The result of the submit action.
"""
call_path = os.path.abspath(sys._getframe(1).f_code.co_filename)
project_path = os.path.abspath(project_description().project_path)
# length /python_codes/ is 14; length .py is 3
entry_module_path = call_path[len(project_path) + 14:-3].replace('/', '.')
namespace = project_config().get_project_name()
translator = get_default_translator()
workflow = translator.translate(graph=default_graph(),
project_desc=project_description())
for job in workflow.jobs.values():
_register_job_meta(workflow_id=workflow.workflow_id, job=job)
_set_entry_module_path(workflow, entry_module_path)
_upload_project_package(workflow)
return proto_to_workflow(get_ai_flow_client().submit_workflow_to_scheduler(
namespace=namespace,
workflow_json=json_utils.dumps(workflow),
workflow_name=workflow_name,
args=args))
def submit_ai_flow(ai_graph: AIGraph = default_graph(),
project_desc: ProjectDesc = ProjectDesc()) -> Optional[int]:
"""
Submit ai flow to schedule.
:param ai_graph: The ai graph constructed from project.
:param project_desc: The project description.
:return: Workflow id.
"""
ex_workflow = generate_workflow(ai_graph, project_desc)
_default_project.upload_project_package(ex_workflow)
return _default_project.submit_workflow(ex_workflow=ex_workflow)
def _generate_airflow_file_text(ai_graph: AIGraph = default_graph(),
project_desc: ProjectDesc = ProjectDesc(),
dag_id=None) -> Optional[str]:
"""
Submit ai flow to schedule.
:param ai_graph: The ai graph constructed from project.
:param project_desc: The project description.
:return: Workflow id.
"""
ex_workflow = generate_workflow(ai_graph, project_desc)
for job in ex_workflow.jobs.values():
register_job_meta(workflow_id=ex_workflow.workflow_id, job=job)
_default_project.upload_project_package(ex_workflow)
return DAGGenerator().generator(ex_workflow, dag_id)
def run(project_path: Text = None,
dag_id: Text = None,
scheduler_type: SchedulerType = SchedulerType.AIFLOW):
"""
Run project under the current project path.
:param project_path: The path of the project path.
:param dag_id: The airflow dag id.
:param scheduler_type: The scheduler type of workflow.
:return: Workflow id or airflow generated code.
"""
project_desc = generate_project_desc(project_path)
if scheduler_type == SchedulerType.AIRFLOW:
return _submit_to_airflow(project_desc=project_desc, dag_id=dag_id)
else:
return submit_ai_flow(ai_graph=default_graph(),
project_desc=project_desc)
def compile_workflow(project_path: Text = None) -> Workflow:
project_desc = generate_project_desc(project_path)
return generate_workflow(ai_graph=default_graph(),
project_desc=project_desc)
