def run_deployment(self):
self._logger.info("Running prepared deployment, {}".format(
self._deploy_dir))
py_version_major = sys.version_info[0]
sys.path.insert(0, self._deploy_dir)
eggs = glob.glob("{}/*py{}*.egg".format(self._deploy_dir,
py_version_major))
if len(eggs) == 0:
self._logger.warning(
"No eggs found for py{}. Trying to find all possible eggs.".
format(py_version_major))
eggs = glob.glob("{}/*.egg".format(self._deploy_dir))
for egg in eggs:
sys.path.insert(0, egg)
pipeline_file = os.path.join(self._deploy_dir,
MLPiper.DEPLOYMENT_PIPELINE)
if not os.path.exists(pipeline_file):
raise Exception(
"Pipeline file not exists! path: {}".format(pipeline_file))
pipeline_runner = Executor() \
.comp_root_path(self._comp_root_path) \
.pipeline_file(open(pipeline_file)) \
.use_color(self._use_color)
if not self._skip_mlpiper_deps:
py_deps = pipeline_runner.all_py_component_dependencies()
if py_deps:
self._install_deps(py_deps)
pipeline_runner.go()
def deps(self, lang):
self._logger.info("Showing dependencies information...")
self.deploy()
pipeline_file = os.path.join(self._deploy_dir,
MLPiper.DEPLOYMENT_PIPELINE)
pipeline_runner = Executor() \
.comp_root_path(self._comp_root_path) \
.pipeline_file(open(pipeline_file)) \
.use_color(self._use_color)
deps = None
if lang == ComponentLanguage.PYTHON:
deps = pipeline_runner.all_py_component_dependencies()
elif lang == ComponentLanguage.R:
deps = pipeline_runner.all_r_component_dependencies()
else:
pass
print("----- Dependencies -----")
if deps:
for dep in sorted(deps):
print(dep)
else:
print(
"No dependencies found for {} components.\nOr there are no {} components in the pipeline."
.format(lang, lang))
def run_deployment(self):
self._logger.info("Running prepared deployment, {}".format(
self._deploy_dir))
py_version_major = sys.version_info[0]
sys.path.insert(0, self._deploy_dir)
eggs = glob.glob("{}/*py{}*.egg".format(self._deploy_dir,
py_version_major))
if len(eggs) == 0:
self._logger.warning(
"No eggs found for py{}. Trying to find all possible eggs.".
format(py_version_major))
eggs = glob.glob("{}/*.egg".format(self._deploy_dir))
for egg in eggs:
sys.path.insert(0, egg)
pipeline_file = os.path.join(self._deploy_dir,
MLPiper.DEPLOYMENT_PIPELINE)
if not os.path.exists(pipeline_file):
raise Exception(
"Pipeline file not exists! path: {}".format(pipeline_file))
pipeline_json = None
with open(pipeline_file, 'r') as f:
pipeline_json = json.load(f)
pipeline_json[json_fields.PIPELINE_SYSTEM_CONFIG_FIELD] \
[json_fields.PIPELINE_SYSTEM_CONFIG_TEST_MODE_PARAM] = self._test_mode
config = ExecutorConfig(pipeline=json.dumps(pipeline_json),
pipeline_file=None,
run_locally=False,
mlcomp_jar=None)
pipeline_runner = Executor(config) \
.comp_root_path(self._comp_root_path) \
.pipeline_file(open(pipeline_file)) \
.use_color(self._use_color) \
.mlcomp_jar(self._mlcomp_jar) \
.standalone(True)
if not self._skip_mlpiper_deps:
py_deps = pipeline_runner.all_py_component_dependencies()
if py_deps:
self._install_deps(py_deps)
try:
pipeline_runner.go()
except KeyError as e:
if str(e).find(java_mapping.MODEL_FILE_SINK_PATH_KEY) != -1:
raise MLPiperException(
"Component in pipeline outputs a model, please provide '--output-model' argument"
)
elif str(e).find(java_mapping.MODEL_FILE_SOURCE_PATH_KEY) != -1:
raise MLPiperException(
"Component in pipeline expects to receive a model, please provide '--input-model' argument"
)
else:
raise
def test_execute_python_connected(self, caplog):
pipeline = {
"name":
"stand_alone_test",
"engineType":
"Generic",
"pipe": [{
"name": "src",
"id": 1,
"type": "string-source",
"parents": [],
"arguments": {
"value": "test-st1-1234"
}
}, {
"name": "sink",
"id": 2,
"type": "string-sink",
"parents": [{
"parent": 1,
"output": 0
}],
"arguments": {
"expected-value": "test-st1-1234"
}
}]
}
self._fix_pipeline(pipeline, None)
config = self._get_executor_config(pipeline)
Executor(config).go()
def test_execute_python_stand_alone_with_exit_1(self):
pipeline = {
"name":
"stand_alone_test",
"engineType":
"Generic",
"pipe": [{
"name": "Hello",
"id": 1,
"type": "test-argument-from-env-var",
"parents": [],
"arguments": {
"arg1": "test-exit-1",
"fromEnvVar2": "test-value2",
},
}]
}
self._fix_pipeline(pipeline, None)
config = self._get_executor_config(pipeline)
passed = 0
try:
Executor(config).go()
except ExecutorException as e:
passed = str(e).startswith("Pipeline called exit(), with code: 1")
assert (passed)
def run(self):
self._logger.info("Deputy starting")
# TODO: move to a thread/separate process so we can track
ret_val = 1
try:
pipeline_runner = Executor(args=None)\
.pipeline_file(self._pipeline_file)\
.mlcomp_jar(self._mlcomp_jar)\
.use_color(self._use_color)
py_deps = pipeline_runner.all_py_component_dependencies()
if py_deps:
PyPackageInstaller(py_deps).install()
r_deps = pipeline_runner.all_r_component_dependencies()
if r_deps:
RPackageInstaller(r_deps).install()
pipeline_runner.go()
ret_val = 0
except Exception as e:
self._logger.info("Got exception while running code: {}".format(e))
exc_type, exc_value, exc_traceback = sys.exc_info()
print("========= Error from code ==========")
traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout)
print("========= Error from code ==========")
ret_val = 1
finally:
self._logger.info("Deputy done - finally block")
return ret_val
def test_execute_r_connected(self):
pipeline = {
"name":
"connected_java_test",
"engineType":
"Generic",
"pipe": [{
"name": "src",
"id": 1,
"type": "string-source",
"parents": [],
"arguments": {
"value": "test-st1-1234"
}
}, {
"name": "infer",
"id": 2,
"type": "test-r-predict-middle",
"parents": [{
"parent": 1,
"output": 0
}],
"arguments": {
"iter": 1,
"expected_input_str": "test-st1-1234"
}
}, {
"name": "sink",
"id": 3,
"type": "string-sink",
"parents": [{
"parent": 2,
"output": 0
}],
"arguments": {
"expected-value": "test-st1-1234"
}
}]
}
model_file = self._gen_model_file()
self._fix_pipeline(pipeline, model_file)
try:
config = self._get_executor_config(pipeline)
Executor(config).go()
finally:
os.remove(model_file)
def test_execute_java_connected_error(self, caplog):
pipeline = {
"name":
"connected_java_test_error",
"engineType":
"Generic",
"pipe": [{
"name": "src",
"id": 1,
"type": "string-source",
"parents": [],
"arguments": {
"value": "test-st1-1234"
}
}, {
"name": "infer",
"id": 2,
"type": "test-java-predict-middle",
"parents": [{
"parent": 1,
"output": 0
}],
"arguments": {
"iter": 1,
"exit_value": -1
}
}, {
"name": "sink",
"id": 3,
"type": "string-sink",
"parents": [{
"parent": 2,
"output": 0
}],
"arguments": {
"expected-value": "test-st1-1234"
}
}]
}
caplog.set_level(logging.INFO)
model_file = self._gen_model_file()
self._fix_pipeline(pipeline, model_file)
with pytest.raises(Exception):
config = self._get_executor_config(pipeline)
Executor(config).go()
os.remove(model_file)
def test_execute_java_connected_multiple_jars(self):
pipeline = {
"name":
"connected_with_multiple_jars_test",
"engineType":
"Generic",
"pipe": [{
"name": "Java",
"id": 1,
"type": "test-java-connected-multiple-jars",
"parents": [],
"arguments": {}
}]
}
system_config = TestPythonEngine.system_config
pipeline["systemConfig"] = system_config
config = self._get_executor_config(pipeline)
Executor(config).go()
def test_execute_python_stand_alone(self):
pipeline = {
"name":
"stand_alone_test",
"engineType":
"Generic",
"pipe": [{
"name": "Hello",
"id": 1,
"type": "hello-world",
"parents": [],
"arguments": {
"arg1": "arg1-value"
}
}]
}
self._fix_pipeline(pipeline, None)
config = self._get_executor_config(pipeline)
Executor(config).go()
def test_execute_python_stand_alone_with_exit_0(self):
pipeline = {
"name":
"stand_alone_test",
"engineType":
"Generic",
"pipe": [{
"name": "Hello",
"id": 1,
"type": "test-argument-from-env-var",
"parents": [],
"arguments": {
"arg1": "test-exit-0",
"fromEnvVar2": "test-value2",
},
}]
}
self._fix_pipeline(pipeline, None)
config = self._get_executor_config(pipeline)
Executor(config).go()
def test_execute_python_stand_alone_with_argument_from_env_var(self):
pipeline = {
"name":
"stand_alone_test",
"engineType":
"Generic",
"pipe": [{
"name": "Hello",
"id": 1,
"type": "test-argument-from-env-var",
"parents": [],
"arguments": {
"arg1": "test-value",
"fromEnvVar2": "test-value2",
},
}]
}
self._fix_pipeline(pipeline, None)
config = self._get_executor_config(pipeline)
os.environ.setdefault("TEST_VAR", "test-value")
os.environ.setdefault("TEST_VAR2", "non test value")
Executor(config).go()
def test_execute_r_stand_alone(self):
pipeline = {
"name":
"stand_alone_test",
"engineType":
"Generic",
"pipe": [{
"name": "R",
"id": 1,
"type": "test-r-predict",
"parents": [],
"arguments": {
"data-file": "/tmp/ddd.data"
}
}]
}
model_file = self._gen_model_file()
self._fix_pipeline(pipeline, model_file)
try:
config = self._get_executor_config(pipeline)
Executor(config).go()
finally:
os.remove(model_file)
def test_execute_java_stand_alone(self):
pipeline = {
"name":
"stand_alone_test",
"engineType":
"Generic",
"pipe": [{
"name": "Java",
"id": 1,
"type": "test-java-standalone",
"parents": [],
"arguments": {
"iter": 1
}
}]
}
model_file = self._gen_model_file()
self._fix_pipeline(pipeline, model_file)
try:
config = self._get_executor_config(pipeline)
Executor(config).go()
finally:
os.remove(model_file)
def test_accumulated_r_deps(self):
with open(TestPythonDeps.pipeline_tmp_file, 'r') as f:
pipeline_runner = Executor(args=None).pipeline_file(f)
deps = pipeline_runner.all_r_component_dependencies()
assert deps == set(expected_r_deps)
