def _add_example_node_to_graph(node: Example,
input_data: Optional[Channel] = None):
config = default_af_job_context().merge_config()
if config.engine == NONE_ENGINE:
default_af_job_context().job_config.engine_name = 'python'
_default_ai_graph.add_node(node)
if input_data is not None:
_default_ai_graph.add_channel(instance_id=node.instance_id,
channel=input_data)
def deploy_to_airflow(project_path: Text = None,
dag_id: Text = None,
default_args=None):
"""
Run project under the current project path.
:param project_path: The description of the project..
:param dag_id: The airflow dag id.
:param default_args:
:return: (airflow dag file path, airflow dag code).
"""
project_desc = generate_project_desc(project_path)
if dag_id is None:
dag_id = default_af_job_context().global_workflow_config.name
if dag_id is None:
dag_id = project_desc.project_name
deploy_path = project_desc.project_config.get_airflow_deploy_path()
if deploy_path is None:
raise Exception("airflow_deploy_path config not set!")
airflow_file_path = deploy_path + '/' + dag_id + '.py'
if os.path.exists(airflow_file_path):
os.remove(airflow_file_path)
generated_code = _generate_airflow_file_text(ai_graph=default_graph(),
project_desc=project_desc,
dag_id=dag_id,
default_args=default_args)
with NamedTemporaryFile(mode='w+t',
prefix=dag_id,
suffix='.py',
dir='/tmp',
delete=False) as f:
f.write(generated_code)
os.rename(f.name, airflow_file_path)
return airflow_file_path, generated_code
def _add_execute_node_to_graph(executor, node, inputs: Union[None, Channel,
List[Channel]]):
config = default_af_job_context().merge_config()
if config.engine == NONE_ENGINE:
if isinstance(executor, CmdExecutor):
default_af_job_context().job_config.engine = 'cmd_line'
else:
default_af_job_context().job_config.engine = 'python'
_default_ai_graph.add_node(node)
if isinstance(inputs, Channel):
_default_ai_graph.add_channel(instance_id=node.instance_id,
channel=inputs)
elif isinstance(inputs, List):
for c in inputs:
_default_ai_graph.add_channel(instance_id=node.instance_id,
channel=c)
def add_node(self, node: AINode):
instance_id = _get_id_generator(self).generate_id(node)
node.set_instance_id(instance_id)
node.config = default_af_job_context().merge_config()
node.properties[ENGINE_NAME] = node.config.engine
self.nodes[instance_id] = node