def all_component_dependencies(self, lang): accumulated_py_deps = set() comps_desc_list = components_desc.ComponentsDesc( pipeline=self.pipeline, comp_root_path=self._comp_root_path).load(extended=True) for comp_desc in comps_desc_list: if comp_desc[json_fields.PIPELINE_LANGUAGE_FIELD] == lang: deps = comp_desc.get(json_fields.COMPONENT_DESC_PYTHON_DEPS, None) if deps: accumulated_py_deps.update(deps) # for Python fetch deps from requirements.txt file if lang == ComponentLanguage.PYTHON: req_file = os.path.join( comp_desc[json_fields.COMPONENT_DESC_ROOT_PATH_FIELD], MLCompConstants.REQUIREMENTS_FILENAME) if os.path.exists(req_file): with open(req_file) as f: content = f.readlines() content = [x.strip() for x in content] accumulated_py_deps.update(content) return accumulated_py_deps
def go(self): """ Actual execution phase """ self._logger.debug("Executor.go()") try: self._init_ml_engine(self.pipeline) comps_desc_list = components_desc.ComponentsDesc(self._ml_engine, self.pipeline, self._comp_root_path)\ .load() self._logger.debug("comp_desc: {}".format(comps_desc_list)) dag = Dag(self.pipeline, comps_desc_list, self._ml_engine).use_color(self._use_color) # Flush stdout so the logs looks a bit in order sys.stdout.flush() system_conf = self.pipeline[ json_fields.PIPELINE_SYSTEM_CONFIG_FIELD] mlops._set_test_mode( system_conf.get( json_fields.PIPELINE_SYSTEM_CONFIG_TEST_MODE_PARAM, False)) ee_conf = self.pipeline.get(json_fields.PIPELINE_EE_CONF_FIELD, dict()) if dag.is_stand_alone: dag.run_single_component_pipeline(system_conf, ee_conf, self._ml_engine) else: dag.run_connected_pipeline(system_conf, ee_conf, self._ml_engine) # This except is intended to catch exit() calls from components. # Do not use exit() in mlpiper code. except SystemExit as e: code = self._parse_exit_code(e.code) error_message = "Pipeline called exit(), with code: {}".format( e.code) traceback_message = traceback.format_exc() if code != 0: self._logger.error("{}\n{}".format(error_message, traceback_message)) # For Py2 put traceback into the exception message if sys.version_info[0] == 2: error_message = "{}\n{}".format(error_message, traceback.format_exc()) raise ExecutorException(error_message) else: self._logger.warning(error_message) except KeyboardInterrupt: # When running from mlpiper tool (standalone) pass finally: sys.stdout.flush() self._logger.info("Done running pipeline (in finally block)") self._cleanup_on_exist() print("End of go")
def all_component_dependencies(self, lang): accumulated_py_deps = set() comps_desc_list = components_desc.ComponentsDesc( pipeline=self.pipeline, comp_root_path=self._comp_root_path).load() for comp_desc in comps_desc_list: if comp_desc[json_fields.PIPELINE_LANGUAGE_FIELD] == lang: deps = comp_desc.get(json_fields.COMPONENT_DESC_PYTHON_DEPS, None) if deps: accumulated_py_deps.update(deps) return accumulated_py_deps