def all_component_dependencies(self, lang):
accumulated_py_deps = set()
comps_desc_list = components_desc.ComponentsDesc(
pipeline=self.pipeline,
comp_root_path=self._comp_root_path).load(extended=True)
for comp_desc in comps_desc_list:
if comp_desc[json_fields.PIPELINE_LANGUAGE_FIELD] == lang:
deps = comp_desc.get(json_fields.COMPONENT_DESC_PYTHON_DEPS,
None)
if deps:
accumulated_py_deps.update(deps)
# for Python fetch deps from requirements.txt file
if lang == ComponentLanguage.PYTHON:
req_file = os.path.join(
comp_desc[json_fields.COMPONENT_DESC_ROOT_PATH_FIELD],
MLCompConstants.REQUIREMENTS_FILENAME)
if os.path.exists(req_file):
with open(req_file) as f:
content = f.readlines()
content = [x.strip() for x in content]
accumulated_py_deps.update(content)
return accumulated_py_deps
def go(self):
"""
Actual execution phase
"""
self._logger.debug("Executor.go()")
try:
self._init_ml_engine(self.pipeline)
comps_desc_list = components_desc.ComponentsDesc(self._ml_engine, self.pipeline, self._comp_root_path)\
.load()
self._logger.debug("comp_desc: {}".format(comps_desc_list))
dag = Dag(self.pipeline, comps_desc_list,
self._ml_engine).use_color(self._use_color)
# Flush stdout so the logs looks a bit in order
sys.stdout.flush()
system_conf = self.pipeline[
json_fields.PIPELINE_SYSTEM_CONFIG_FIELD]
mlops._set_test_mode(
system_conf.get(
json_fields.PIPELINE_SYSTEM_CONFIG_TEST_MODE_PARAM, False))
ee_conf = self.pipeline.get(json_fields.PIPELINE_EE_CONF_FIELD,
dict())
if dag.is_stand_alone:
dag.run_single_component_pipeline(system_conf, ee_conf,
self._ml_engine)
else:
dag.run_connected_pipeline(system_conf, ee_conf,
self._ml_engine)
# This except is intended to catch exit() calls from components.
# Do not use exit() in mlpiper code.
except SystemExit as e:
code = self._parse_exit_code(e.code)
error_message = "Pipeline called exit(), with code: {}".format(
e.code)
traceback_message = traceback.format_exc()
if code != 0:
self._logger.error("{}\n{}".format(error_message,
traceback_message))
# For Py2 put traceback into the exception message
if sys.version_info[0] == 2:
error_message = "{}\n{}".format(error_message,
traceback.format_exc())
raise ExecutorException(error_message)
else:
self._logger.warning(error_message)
except KeyboardInterrupt:
# When running from mlpiper tool (standalone)
pass
finally:
sys.stdout.flush()
self._logger.info("Done running pipeline (in finally block)")
self._cleanup_on_exist()
print("End of go")
def all_component_dependencies(self, lang):
accumulated_py_deps = set()
comps_desc_list = components_desc.ComponentsDesc(
pipeline=self.pipeline,
comp_root_path=self._comp_root_path).load()
for comp_desc in comps_desc_list:
if comp_desc[json_fields.PIPELINE_LANGUAGE_FIELD] == lang:
deps = comp_desc.get(json_fields.COMPONENT_DESC_PYTHON_DEPS,
None)
if deps:
accumulated_py_deps.update(deps)
return accumulated_py_deps
