def deploy_to_airflow(project_path: Text = None, dag_id: Text = None, default_args=None): """ Run project under the current project path. :param project_path: The description of the project.. :param dag_id: The airflow dag id. :param default_args: :return: (airflow dag file path, airflow dag code). """ project_desc = generate_project_desc(project_path) if dag_id is None: dag_id = default_af_job_context().global_workflow_config.name if dag_id is None: dag_id = project_desc.project_name deploy_path = project_desc.project_config.get_airflow_deploy_path() if deploy_path is None: raise Exception("airflow_deploy_path config not set!") airflow_file_path = deploy_path + '/' + dag_id + '.py' if os.path.exists(airflow_file_path): os.remove(airflow_file_path) generated_code = _generate_airflow_file_text(ai_graph=default_graph(), project_desc=project_desc, dag_id=dag_id, default_args=default_args) with NamedTemporaryFile(mode='w+t', prefix=dag_id, suffix='.py', dir='/tmp', delete=False) as f: f.write(generated_code) os.rename(f.name, airflow_file_path) return airflow_file_path, generated_code
def submit_workflow(workflow_name: Text = None, args: Dict = None) -> WorkflowInfo: """ Submit the ai flow workflow to the scheduler. :param workflow_name: The ai flow workflow identify. :param args: The arguments of the submit action. :return: The result of the submit action. """ call_path = os.path.abspath(sys._getframe(1).f_code.co_filename) project_path = os.path.abspath(project_description().project_path) # length /python_codes/ is 14; length .py is 3 entry_module_path = call_path[len(project_path) + 14:-3].replace('/', '.') namespace = project_config().get_project_name() translator = get_default_translator() workflow = translator.translate(graph=default_graph(), project_desc=project_description()) for job in workflow.jobs.values(): _register_job_meta(workflow_id=workflow.workflow_id, job=job) _set_entry_module_path(workflow, entry_module_path) _upload_project_package(workflow) return proto_to_workflow(get_ai_flow_client().submit_workflow_to_scheduler( namespace=namespace, workflow_json=json_utils.dumps(workflow), workflow_name=workflow_name, args=args))
def submit_ai_flow(ai_graph: AIGraph = default_graph(), project_desc: ProjectDesc = ProjectDesc()) -> Optional[int]: """ Submit ai flow to schedule. :param ai_graph: The ai graph constructed from project. :param project_desc: The project description. :return: Workflow id. """ ex_workflow = generate_workflow(ai_graph, project_desc) _default_project.upload_project_package(ex_workflow) return _default_project.submit_workflow(ex_workflow=ex_workflow)
def _generate_airflow_file_text(ai_graph: AIGraph = default_graph(), project_desc: ProjectDesc = ProjectDesc(), dag_id=None) -> Optional[str]: """ Submit ai flow to schedule. :param ai_graph: The ai graph constructed from project. :param project_desc: The project description. :return: Workflow id. """ ex_workflow = generate_workflow(ai_graph, project_desc) for job in ex_workflow.jobs.values(): register_job_meta(workflow_id=ex_workflow.workflow_id, job=job) _default_project.upload_project_package(ex_workflow) return DAGGenerator().generator(ex_workflow, dag_id)
def run(project_path: Text = None, dag_id: Text = None, scheduler_type: SchedulerType = SchedulerType.AIFLOW): """ Run project under the current project path. :param project_path: The path of the project path. :param dag_id: The airflow dag id. :param scheduler_type: The scheduler type of workflow. :return: Workflow id or airflow generated code. """ project_desc = generate_project_desc(project_path) if scheduler_type == SchedulerType.AIRFLOW: return _submit_to_airflow(project_desc=project_desc, dag_id=dag_id) else: return submit_ai_flow(ai_graph=default_graph(), project_desc=project_desc)
def compile_workflow(project_path: Text = None) -> Workflow: project_desc = generate_project_desc(project_path) return generate_workflow(ai_graph=default_graph(), project_desc=project_desc)