def run_deployment(self): self._logger.info("Running prepared deployment, {}".format( self._deploy_dir)) py_version_major = sys.version_info[0] sys.path.insert(0, self._deploy_dir) eggs = glob.glob("{}/*py{}*.egg".format(self._deploy_dir, py_version_major)) if len(eggs) == 0: self._logger.warning( "No eggs found for py{}. Trying to find all possible eggs.". format(py_version_major)) eggs = glob.glob("{}/*.egg".format(self._deploy_dir)) for egg in eggs: sys.path.insert(0, egg) pipeline_file = os.path.join(self._deploy_dir, MLPiper.DEPLOYMENT_PIPELINE) if not os.path.exists(pipeline_file): raise Exception( "Pipeline file not exists! path: {}".format(pipeline_file)) pipeline_runner = Executor() \ .comp_root_path(self._comp_root_path) \ .pipeline_file(open(pipeline_file)) \ .use_color(self._use_color) if not self._skip_mlpiper_deps: py_deps = pipeline_runner.all_py_component_dependencies() if py_deps: self._install_deps(py_deps) pipeline_runner.go()
def deps(self, lang): self._logger.info("Showing dependencies information...") self.deploy() pipeline_file = os.path.join(self._deploy_dir, MLPiper.DEPLOYMENT_PIPELINE) pipeline_runner = Executor() \ .comp_root_path(self._comp_root_path) \ .pipeline_file(open(pipeline_file)) \ .use_color(self._use_color) deps = None if lang == ComponentLanguage.PYTHON: deps = pipeline_runner.all_py_component_dependencies() elif lang == ComponentLanguage.R: deps = pipeline_runner.all_r_component_dependencies() else: pass print("----- Dependencies -----") if deps: for dep in sorted(deps): print(dep) else: print( "No dependencies found for {} components.\nOr there are no {} components in the pipeline." .format(lang, lang))
def run_deployment(self): self._logger.info("Running prepared deployment, {}".format( self._deploy_dir)) py_version_major = sys.version_info[0] sys.path.insert(0, self._deploy_dir) eggs = glob.glob("{}/*py{}*.egg".format(self._deploy_dir, py_version_major)) if len(eggs) == 0: self._logger.warning( "No eggs found for py{}. Trying to find all possible eggs.". format(py_version_major)) eggs = glob.glob("{}/*.egg".format(self._deploy_dir)) for egg in eggs: sys.path.insert(0, egg) pipeline_file = os.path.join(self._deploy_dir, MLPiper.DEPLOYMENT_PIPELINE) if not os.path.exists(pipeline_file): raise Exception( "Pipeline file not exists! path: {}".format(pipeline_file)) pipeline_json = None with open(pipeline_file, 'r') as f: pipeline_json = json.load(f) pipeline_json[json_fields.PIPELINE_SYSTEM_CONFIG_FIELD] \ [json_fields.PIPELINE_SYSTEM_CONFIG_TEST_MODE_PARAM] = self._test_mode config = ExecutorConfig(pipeline=json.dumps(pipeline_json), pipeline_file=None, run_locally=False, mlcomp_jar=None) pipeline_runner = Executor(config) \ .comp_root_path(self._comp_root_path) \ .pipeline_file(open(pipeline_file)) \ .use_color(self._use_color) \ .mlcomp_jar(self._mlcomp_jar) \ .standalone(True) if not self._skip_mlpiper_deps: py_deps = pipeline_runner.all_py_component_dependencies() if py_deps: self._install_deps(py_deps) try: pipeline_runner.go() except KeyError as e: if str(e).find(java_mapping.MODEL_FILE_SINK_PATH_KEY) != -1: raise MLPiperException( "Component in pipeline outputs a model, please provide '--output-model' argument" ) elif str(e).find(java_mapping.MODEL_FILE_SOURCE_PATH_KEY) != -1: raise MLPiperException( "Component in pipeline expects to receive a model, please provide '--input-model' argument" ) else: raise
def test_execute_python_connected(self, caplog): pipeline = { "name": "stand_alone_test", "engineType": "Generic", "pipe": [{ "name": "src", "id": 1, "type": "string-source", "parents": [], "arguments": { "value": "test-st1-1234" } }, { "name": "sink", "id": 2, "type": "string-sink", "parents": [{ "parent": 1, "output": 0 }], "arguments": { "expected-value": "test-st1-1234" } }] } self._fix_pipeline(pipeline, None) config = self._get_executor_config(pipeline) Executor(config).go()
def test_execute_python_stand_alone_with_exit_1(self): pipeline = { "name": "stand_alone_test", "engineType": "Generic", "pipe": [{ "name": "Hello", "id": 1, "type": "test-argument-from-env-var", "parents": [], "arguments": { "arg1": "test-exit-1", "fromEnvVar2": "test-value2", }, }] } self._fix_pipeline(pipeline, None) config = self._get_executor_config(pipeline) passed = 0 try: Executor(config).go() except ExecutorException as e: passed = str(e).startswith("Pipeline called exit(), with code: 1") assert (passed)
def run(self): self._logger.info("Deputy starting") # TODO: move to a thread/separate process so we can track ret_val = 1 try: pipeline_runner = Executor(args=None)\ .pipeline_file(self._pipeline_file)\ .mlcomp_jar(self._mlcomp_jar)\ .use_color(self._use_color) py_deps = pipeline_runner.all_py_component_dependencies() if py_deps: PyPackageInstaller(py_deps).install() r_deps = pipeline_runner.all_r_component_dependencies() if r_deps: RPackageInstaller(r_deps).install() pipeline_runner.go() ret_val = 0 except Exception as e: self._logger.info("Got exception while running code: {}".format(e)) exc_type, exc_value, exc_traceback = sys.exc_info() print("========= Error from code ==========") traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout) print("========= Error from code ==========") ret_val = 1 finally: self._logger.info("Deputy done - finally block") return ret_val
def test_execute_r_connected(self): pipeline = { "name": "connected_java_test", "engineType": "Generic", "pipe": [{ "name": "src", "id": 1, "type": "string-source", "parents": [], "arguments": { "value": "test-st1-1234" } }, { "name": "infer", "id": 2, "type": "test-r-predict-middle", "parents": [{ "parent": 1, "output": 0 }], "arguments": { "iter": 1, "expected_input_str": "test-st1-1234" } }, { "name": "sink", "id": 3, "type": "string-sink", "parents": [{ "parent": 2, "output": 0 }], "arguments": { "expected-value": "test-st1-1234" } }] } model_file = self._gen_model_file() self._fix_pipeline(pipeline, model_file) try: config = self._get_executor_config(pipeline) Executor(config).go() finally: os.remove(model_file)
def test_execute_java_connected_error(self, caplog): pipeline = { "name": "connected_java_test_error", "engineType": "Generic", "pipe": [{ "name": "src", "id": 1, "type": "string-source", "parents": [], "arguments": { "value": "test-st1-1234" } }, { "name": "infer", "id": 2, "type": "test-java-predict-middle", "parents": [{ "parent": 1, "output": 0 }], "arguments": { "iter": 1, "exit_value": -1 } }, { "name": "sink", "id": 3, "type": "string-sink", "parents": [{ "parent": 2, "output": 0 }], "arguments": { "expected-value": "test-st1-1234" } }] } caplog.set_level(logging.INFO) model_file = self._gen_model_file() self._fix_pipeline(pipeline, model_file) with pytest.raises(Exception): config = self._get_executor_config(pipeline) Executor(config).go() os.remove(model_file)
def test_execute_java_connected_multiple_jars(self): pipeline = { "name": "connected_with_multiple_jars_test", "engineType": "Generic", "pipe": [{ "name": "Java", "id": 1, "type": "test-java-connected-multiple-jars", "parents": [], "arguments": {} }] } system_config = TestPythonEngine.system_config pipeline["systemConfig"] = system_config config = self._get_executor_config(pipeline) Executor(config).go()
def test_execute_python_stand_alone(self): pipeline = { "name": "stand_alone_test", "engineType": "Generic", "pipe": [{ "name": "Hello", "id": 1, "type": "hello-world", "parents": [], "arguments": { "arg1": "arg1-value" } }] } self._fix_pipeline(pipeline, None) config = self._get_executor_config(pipeline) Executor(config).go()
def test_execute_python_stand_alone_with_exit_0(self): pipeline = { "name": "stand_alone_test", "engineType": "Generic", "pipe": [{ "name": "Hello", "id": 1, "type": "test-argument-from-env-var", "parents": [], "arguments": { "arg1": "test-exit-0", "fromEnvVar2": "test-value2", }, }] } self._fix_pipeline(pipeline, None) config = self._get_executor_config(pipeline) Executor(config).go()
def test_execute_python_stand_alone_with_argument_from_env_var(self): pipeline = { "name": "stand_alone_test", "engineType": "Generic", "pipe": [{ "name": "Hello", "id": 1, "type": "test-argument-from-env-var", "parents": [], "arguments": { "arg1": "test-value", "fromEnvVar2": "test-value2", }, }] } self._fix_pipeline(pipeline, None) config = self._get_executor_config(pipeline) os.environ.setdefault("TEST_VAR", "test-value") os.environ.setdefault("TEST_VAR2", "non test value") Executor(config).go()
def test_execute_r_stand_alone(self): pipeline = { "name": "stand_alone_test", "engineType": "Generic", "pipe": [{ "name": "R", "id": 1, "type": "test-r-predict", "parents": [], "arguments": { "data-file": "/tmp/ddd.data" } }] } model_file = self._gen_model_file() self._fix_pipeline(pipeline, model_file) try: config = self._get_executor_config(pipeline) Executor(config).go() finally: os.remove(model_file)
def test_execute_java_stand_alone(self): pipeline = { "name": "stand_alone_test", "engineType": "Generic", "pipe": [{ "name": "Java", "id": 1, "type": "test-java-standalone", "parents": [], "arguments": { "iter": 1 } }] } model_file = self._gen_model_file() self._fix_pipeline(pipeline, model_file) try: config = self._get_executor_config(pipeline) Executor(config).go() finally: os.remove(model_file)
def test_accumulated_r_deps(self): with open(TestPythonDeps.pipeline_tmp_file, 'r') as f: pipeline_runner = Executor(args=None).pipeline_file(f) deps = pipeline_runner.all_r_component_dependencies() assert deps == set(expected_r_deps)