def run_deployment(self): self._logger.info("Running prepared deployment, {}".format( self._deploy_dir)) py_version_major = sys.version_info[0] sys.path.insert(0, self._deploy_dir) eggs = glob.glob("{}/*py{}*.egg".format(self._deploy_dir, py_version_major)) if len(eggs) == 0: self._logger.warning( "No eggs found for py{}. Trying to find all possible eggs.". format(py_version_major)) eggs = glob.glob("{}/*.egg".format(self._deploy_dir)) for egg in eggs: sys.path.insert(0, egg) pipeline_file = os.path.join(self._deploy_dir, MLPiper.DEPLOYMENT_PIPELINE) if not os.path.exists(pipeline_file): raise Exception( "Pipeline file not exists! path: {}".format(pipeline_file)) pipeline_runner = Executor() \ .comp_root_path(self._comp_root_path) \ .pipeline_file(open(pipeline_file)) \ .use_color(self._use_color) if not self._skip_mlpiper_deps: py_deps = pipeline_runner.all_py_component_dependencies() if py_deps: self._install_deps(py_deps) pipeline_runner.go()
def deps(self, lang): self._logger.info("Showing dependencies information...") self.deploy() pipeline_file = os.path.join(self._deploy_dir, MLPiper.DEPLOYMENT_PIPELINE) pipeline_runner = Executor() \ .comp_root_path(self._comp_root_path) \ .pipeline_file(open(pipeline_file)) \ .use_color(self._use_color) deps = None if lang == ComponentLanguage.PYTHON: deps = pipeline_runner.all_py_component_dependencies() elif lang == ComponentLanguage.R: deps = pipeline_runner.all_r_component_dependencies() else: pass print("----- Dependencies -----") if deps: for dep in sorted(deps): print(dep) else: print( "No dependencies found for {} components.\nOr there are no {} components in the pipeline." .format(lang, lang))
def run(self): self._logger.info("Deputy starting") # TODO: move to a thread/separate process so we can track ret_val = 1 try: pipeline_runner = Executor(args=None)\ .pipeline_file(self._pipeline_file)\ .mlcomp_jar(self._mlcomp_jar)\ .use_color(self._use_color) py_deps = pipeline_runner.all_py_component_dependencies() if py_deps: PyPackageInstaller(py_deps).install() r_deps = pipeline_runner.all_r_component_dependencies() if r_deps: RPackageInstaller(r_deps).install() pipeline_runner.go() ret_val = 0 except Exception as e: self._logger.info("Got exception while running code: {}".format(e)) exc_type, exc_value, exc_traceback = sys.exc_info() print("========= Error from code ==========") traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout) print("========= Error from code ==========") ret_val = 1 finally: self._logger.info("Deputy done - finally block") return ret_val
def run_deployment(self): self._logger.info("Running prepared deployment, {}".format( self._deploy_dir)) py_version_major = sys.version_info[0] sys.path.insert(0, self._deploy_dir) eggs = glob.glob("{}/*py{}*.egg".format(self._deploy_dir, py_version_major)) if len(eggs) == 0: self._logger.warning( "No eggs found for py{}. Trying to find all possible eggs.". format(py_version_major)) eggs = glob.glob("{}/*.egg".format(self._deploy_dir)) for egg in eggs: sys.path.insert(0, egg) pipeline_file = os.path.join(self._deploy_dir, MLPiper.DEPLOYMENT_PIPELINE) if not os.path.exists(pipeline_file): raise Exception( "Pipeline file not exists! path: {}".format(pipeline_file)) pipeline_json = None with open(pipeline_file, 'r') as f: pipeline_json = json.load(f) pipeline_json[json_fields.PIPELINE_SYSTEM_CONFIG_FIELD] \ [json_fields.PIPELINE_SYSTEM_CONFIG_TEST_MODE_PARAM] = self._test_mode config = ExecutorConfig(pipeline=json.dumps(pipeline_json), pipeline_file=None, run_locally=False, mlcomp_jar=None) pipeline_runner = Executor(config) \ .comp_root_path(self._comp_root_path) \ .pipeline_file(open(pipeline_file)) \ .use_color(self._use_color) \ .mlcomp_jar(self._mlcomp_jar) \ .standalone(True) if not self._skip_mlpiper_deps: py_deps = pipeline_runner.all_py_component_dependencies() if py_deps: self._install_deps(py_deps) try: pipeline_runner.go() except KeyError as e: if str(e).find(java_mapping.MODEL_FILE_SINK_PATH_KEY) != -1: raise MLPiperException( "Component in pipeline outputs a model, please provide '--output-model' argument" ) elif str(e).find(java_mapping.MODEL_FILE_SOURCE_PATH_KEY) != -1: raise MLPiperException( "Component in pipeline expects to receive a model, please provide '--input-model' argument" ) else: raise
def test_accumulated_python_deps(self): with open(TestPythonDeps.pipeline_tmp_file, 'r') as f: pipeline_runner = Executor(args=None).pipeline_file(f) deps = pipeline_runner.all_py_component_dependencies() assert deps == set(expected_py_deps)