def _setup_model_env(self, model_filepath):
self._logger.info("Setup model env with: {}".format(model_filepath))
self._model_env = ModelEnv(model_filepath)
if os.path.isfile(model_filepath):
self._logger.info("Rename model file path to {}".format(
self._model_env.model_filepath))
os.rename(model_filepath, self._model_env.model_filepath)
Metadata(self._model_env.model_filepath).save(
self._model_env.metadata_filepath)
class ModelFetcher(BgActor):
POLLING_INTERVAL_SEC = 10.0
def __init__(self, mlops, model_filepath, ml_engine):
super(ModelFetcher, self).__init__(mlops, ml_engine,
ModelFetcher.POLLING_INTERVAL_SEC)
self._uuid = uuid.uuid4()
self._current_model = self._mlops.current_model()
self._logger.info("Current model: {}".format(self._current_model))
self._model_env = None
self._setup_model_env(model_filepath)
def _setup_model_env(self, model_filepath):
self._logger.info("Setup model env with: {}".format(model_filepath))
self._model_env = ModelEnv(model_filepath)
if os.path.isfile(model_filepath):
self._logger.info("Rename model file path to {}".format(
self._model_env.model_filepath))
os.rename(model_filepath, self._model_env.model_filepath)
Metadata(self._model_env.model_filepath).save(
self._model_env.metadata_filepath)
# Overloaded function
def _do_repetitive_work(self):
try:
last_approved_model = self._mlops.get_last_approved_model()
self._logger.debug("Last approved model: {}, uuid: {}".format(
last_approved_model.get_id() if last_approved_model else None,
self._uuid))
if last_approved_model and (
not self._current_model
or last_approved_model != self._current_model):
self._current_model = last_approved_model
self._download_and_signal()
except Exception as ex:
self._logger.error(
"Failed fetch last approved model from server! {}\n{}".format(
traceback.format_exc(), ex))
def _download_and_signal(self):
self._logger.info("New model is about to be downloaded: {}".format(
self._current_model))
self._current_model.download(self._model_env.model_filepath)
Metadata(self._model_env.model_filepath).save(
self._model_env.metadata_filepath)
self._signal()
def _signal(self):
self._model_env.touch_sync()
def _setup(self, pipeline_name, monitor_info):
target_path = tempfile.mkdtemp(dir=ComponentConstants.TMP_RESTFUL_ROOT,
prefix=ComponentConstants.TMP_RESTFUL_DIR_PREFIX)
os.chmod(target_path, 0o777)
shared_conf = {
SharedConstants.TARGET_PATH_KEY: target_path,
SharedConstants.SOCK_FILENAME_KEY: UwsgiConstants.SOCK_FILENAME,
SharedConstants.STATS_SOCK_FILENAME_KEY: UwsgiConstants.STATS_SOCK_FILENAME
}
log_format = self._params.get(ComponentConstants.LOG_FORMAT_KEY, ComponentConstants.DEFAULT_LOG_FORMAT)
log_level_param = self._params.get(ComponentConstants.LOG_LEVEL_KEY, ComponentConstants.DEFAULT_LOG_LEVEL).lower()
log_level = constants.LOG_LEVELS.get(log_level_param, logging.INFO)
self._logger.debug("log_level_param: {}, log_level: {}, level_constants: {}"
.format(log_level_param, log_level, constants.LOG_LEVELS))
stats_reporting_interval_sec = self._params.get(ComponentConstants.STATS_REPORTING_INTERVAL_SEC,
ComponentConstants.DEFAULT_STATS_REPORTING_INTERVAL_SEC)
model_filepath_key = java_mapping.RESERVED_KEYS[ComponentConstants.INPUT_MODEL_TAG_NAME]
self._params[model_filepath_key] = ModelEnv(self._params[model_filepath_key]).model_filepath
uwsgi_entry_point_conf = {
UwsgiConstants.RESTFUL_COMP_MODULE_KEY: self.__module__,
UwsgiConstants.RESTFUL_COMP_CLS_KEY: self.__class__.__name__,
ComponentConstants.LOG_FORMAT_KEY: log_format,
ComponentConstants.LOG_LEVEL_KEY: log_level,
ComponentConstants.STATS_REPORTING_INTERVAL_SEC: stats_reporting_interval_sec,
UwsgiConstants.PARAMS_KEY: self._params,
UwsgiConstants.PIPELINE_NAME_KEY: pipeline_name,
UwsgiConstants.MODEL_PATH_KEY: self._params[model_filepath_key],
ComponentConstants.UWSGI_DISABLE_LOGGING_KEY:
parameter.str2bool(self._params.get(ComponentConstants.UWSGI_DISABLE_LOGGING_KEY,
ComponentConstants.DEFAULT_UWSGI_DISABLE_LOGGING)),
ComponentConstants.METRICS_KEY: Metric.metrics()
}
self._logger.debug("uwsgi_entry_point_conf: {}".format(uwsgi_entry_point_conf))
nginx_conf = {
ComponentConstants.HOST_KEY: ComponentConstants.DEFAULT_HOST,
ComponentConstants.PORT_KEY: self._params[ComponentConstants.PORT_KEY],
NginxConstants.DISABLE_ACCESS_LOG_KEY: log_level != logging.DEBUG
}
self._logger.debug("nginx_conf: {}".format(nginx_conf))
self._dry_run = parameter.str2bool(self._params.get(ComponentConstants.DRY_RUN_KEY,
ComponentConstants.DEFAULT_DRY_RUN))
if self._dry_run:
self._logger.warning("\n\n" + 80 * '#' + "\n" + 25 * " " + "Running in DRY RUN mode\n" + 80 * '#')
self._dry_run = parameter.str2bool(self._params.get(ComponentConstants.DRY_RUN_KEY, ComponentConstants.DEFAULT_DRY_RUN))
self._wsgi_broker = UwsgiBroker(self._ml_engine, self._dry_run) \
.setup_and_run(shared_conf, uwsgi_entry_point_conf, monitor_info)
self._nginx_broker = NginxBroker(self._ml_engine, self._dry_run) \
.setup_and_run(shared_conf, nginx_conf)
def __init__(self, model_filepath, standalone=False):
self._model_env = ModelEnv(model_filepath, standalone)
def __init__(self, model_filepath):
self._model_env = ModelEnv(model_filepath)