def run_deployment(self):
self._logger.info("Running prepared deployment, {}".format(
self._deploy_dir))
py_version_major = sys.version_info[0]
sys.path.insert(0, self._deploy_dir)
eggs = glob.glob("{}/*py{}*.egg".format(self._deploy_dir,
py_version_major))
if len(eggs) == 0:
self._logger.warning(
"No eggs found for py{}. Trying to find all possible eggs.".
format(py_version_major))
eggs = glob.glob("{}/*.egg".format(self._deploy_dir))
for egg in eggs:
sys.path.insert(0, egg)
pipeline_file = os.path.join(self._deploy_dir,
MLPiper.DEPLOYMENT_PIPELINE)
if not os.path.exists(pipeline_file):
raise Exception(
"Pipeline file not exists! path: {}".format(pipeline_file))
pipeline_runner = Executor() \
.comp_root_path(self._comp_root_path) \
.pipeline_file(open(pipeline_file)) \
.use_color(self._use_color)
if not self._skip_mlpiper_deps:
py_deps = pipeline_runner.all_py_component_dependencies()
if py_deps:
self._install_deps(py_deps)
pipeline_runner.go()
def deps(self, lang):
self._logger.info("Showing dependencies information...")
self.deploy()
pipeline_file = os.path.join(self._deploy_dir,
MLPiper.DEPLOYMENT_PIPELINE)
pipeline_runner = Executor() \
.comp_root_path(self._comp_root_path) \
.pipeline_file(open(pipeline_file)) \
.use_color(self._use_color)
deps = None
if lang == ComponentLanguage.PYTHON:
deps = pipeline_runner.all_py_component_dependencies()
elif lang == ComponentLanguage.R:
deps = pipeline_runner.all_r_component_dependencies()
else:
pass
print("----- Dependencies -----")
if deps:
for dep in sorted(deps):
print(dep)
else:
print(
"No dependencies found for {} components.\nOr there are no {} components in the pipeline."
.format(lang, lang))
def run(self):
self._logger.info("Deputy starting")
# TODO: move to a thread/separate process so we can track
ret_val = 1
try:
pipeline_runner = Executor(args=None)\
.pipeline_file(self._pipeline_file)\
.mlcomp_jar(self._mlcomp_jar)\
.use_color(self._use_color)
py_deps = pipeline_runner.all_py_component_dependencies()
if py_deps:
PyPackageInstaller(py_deps).install()
r_deps = pipeline_runner.all_r_component_dependencies()
if r_deps:
RPackageInstaller(r_deps).install()
pipeline_runner.go()
ret_val = 0
except Exception as e:
self._logger.info("Got exception while running code: {}".format(e))
exc_type, exc_value, exc_traceback = sys.exc_info()
print("========= Error from code ==========")
traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout)
print("========= Error from code ==========")
ret_val = 1
finally:
self._logger.info("Deputy done - finally block")
return ret_val
def run_deployment(self):
self._logger.info("Running prepared deployment, {}".format(
self._deploy_dir))
py_version_major = sys.version_info[0]
sys.path.insert(0, self._deploy_dir)
eggs = glob.glob("{}/*py{}*.egg".format(self._deploy_dir,
py_version_major))
if len(eggs) == 0:
self._logger.warning(
"No eggs found for py{}. Trying to find all possible eggs.".
format(py_version_major))
eggs = glob.glob("{}/*.egg".format(self._deploy_dir))
for egg in eggs:
sys.path.insert(0, egg)
pipeline_file = os.path.join(self._deploy_dir,
MLPiper.DEPLOYMENT_PIPELINE)
if not os.path.exists(pipeline_file):
raise Exception(
"Pipeline file not exists! path: {}".format(pipeline_file))
pipeline_json = None
with open(pipeline_file, 'r') as f:
pipeline_json = json.load(f)
pipeline_json[json_fields.PIPELINE_SYSTEM_CONFIG_FIELD] \
[json_fields.PIPELINE_SYSTEM_CONFIG_TEST_MODE_PARAM] = self._test_mode
config = ExecutorConfig(pipeline=json.dumps(pipeline_json),
pipeline_file=None,
run_locally=False,
mlcomp_jar=None)
pipeline_runner = Executor(config) \
.comp_root_path(self._comp_root_path) \
.pipeline_file(open(pipeline_file)) \
.use_color(self._use_color) \
.mlcomp_jar(self._mlcomp_jar) \
.standalone(True)
if not self._skip_mlpiper_deps:
py_deps = pipeline_runner.all_py_component_dependencies()
if py_deps:
self._install_deps(py_deps)
try:
pipeline_runner.go()
except KeyError as e:
if str(e).find(java_mapping.MODEL_FILE_SINK_PATH_KEY) != -1:
raise MLPiperException(
"Component in pipeline outputs a model, please provide '--output-model' argument"
)
elif str(e).find(java_mapping.MODEL_FILE_SOURCE_PATH_KEY) != -1:
raise MLPiperException(
"Component in pipeline expects to receive a model, please provide '--input-model' argument"
)
else:
raise
def test_accumulated_python_deps(self):
with open(TestPythonDeps.pipeline_tmp_file, 'r') as f:
pipeline_runner = Executor(args=None).pipeline_file(f)
deps = pipeline_runner.all_py_component_dependencies()
assert deps == set(expected_py_deps)
