def transform(self, initParams): # noqa: E501
"""
Performs the transforms defined for this plugin
:param initParams: A list of directory paths where input files can be found.
:type initParams: dict | bytes
:rtype: None
"""
logger.debug("transform called")
try:
if not isinstance(
initParams,
TransformSpecificationInitParams) and cx.request.is_json:
initParams = mistk.data.utils.deserialize_model(
cx.request.get_json(), TransformSpecificationInitParams)
assert isinstance(initParams, TransformSpecificationInitParams)
task = TransformPluginTask(operation='transform',
parameters={
"inputDirs":
initParams.input_datasets,
"outputDir":
initParams.output_dataset,
"properties": initParams.properties
})
except RuntimeError as inst:
msg = "Error during transform. %s" % str(inst)
logger.exception(msg)
return ServiceError(500, msg), 500
self.add_task(task)
def initialize_model(self, initializationParameters):
"""
Creates and returns an Task which initializes the model with the
optional parameters provided
:param initializationParameters: The parameters used for initialization
:return: The created Task object
"""
logger.debug("Initialize model called")
try:
params = initializationParameters
if not isinstance(params, InitParams) and cx.request.is_json:
params = datautils.deserialize_model(cx.request.get_json(), InitParams)
assert isinstance(params, InitParams)
task = ModelInstanceTask(operation='initialize',
parameters={"objectives": params.objectives,
"props": params.model_properties,
"hparams": params.hyperparameters})
logger.debug('Created initialize model task', extra={'model_task': task})
except RuntimeError as inst:
msg = "Error during model initialization: %s" % str(inst)
logger.exception(msg)
return ServiceError(500, msg), 500
self.add_task(task)
def get_plugin(self, name):
try:
importlib.invalidate_caches()
return importlib.import_module(name)
except Exception as e:
logger.exception("Exception importing plugin module " + name)
return None
def update_state(self, state=None, payload=None):
"""
Updates the state of the EvaluationEndpointService
:param state: The new state. If not given, the current state will be used.
:param payload: Additional data to attach to the state
"""
try:
with self._status_lock.writer_lock:
state = state or self._status.state
ver = self._status.object_info.resource_version + 1
info = ObjectInfo('EvaluationInstanceStatus',
resource_version=ver)
self._status = EvaluationInstanceStatus(info,
state=state,
payload=payload)
watch_manager.notify_watch('status', item=self._status)
except RuntimeError as inst:
msg = "Runtime Error while updating state of EvaluationEndpointService: %s" % str(
inst)
logger.exception(msg)
return ServiceError(500, msg), 500
except Exception as ex:
msg = "Exception while updating state of EvaluationPluginService: %s" % str(
ex)
logger.exception(msg)
return ServiceError(500, msg), 500
def get_status(self, watch=None, resourceVersion=None): # noqa: E501
"""
Retrieves the status of the evaluation plugin
:rtype: EvaluationInstanceStatus
"""
logger.debug("Get status called")
try:
with self._status_lock.reader_lock:
if watch:
return Response(watch_manager.watch(
'status', resourceVersion, self._status),
mimetype="application/json")
else:
return self._status
except RuntimeError as inst:
msg = "Runtime Error while retrieving status of evaluation plugin: %s" % str(
inst)
logger.exception(msg)
return ServiceError(500, msg), 500
except Exception as ex:
msg = "Exception while retrieving status of evaluation plugin: %s" % str(
ex)
logger.exception(msg)
return ServiceError(500, msg), 500
return 'do some magic!'
def evaluate(self, initParams): # noqa: E501
"""
Performs the evaluation defined for this plugin
:param initParams: Init Parameters for the evaluation. Based on EvaluationSpecificationInitParams specification
:type initParams: dict | bytes
:rtype: None
"""
logger.debug("evaluation called")
try:
if not isinstance(initParams, EvaluationSpecificationInitParams) and cx.request.is_json:
initParams = mistk.data.utils.deserialize_model(cx.request.get_json(), EvaluationSpecificationInitParams)
assert isinstance(initParams, EvaluationSpecificationInitParams)
task = EvaluationPluginTask(operation='evaluate',
parameters={"ground_truth_path": initParams.ground_truth_path,
"input_data_path": initParams.input_data_path,
"evaluation_input_format": initParams.evaluation_input_format,
"evaluation_path": initParams.evaluation_path,
"assessment_type": initParams.assessment_type,
"metrics": initParams.metrics,
"properties": initParams.properties})
except RuntimeError as inst:
msg = "Runtime Error while performing evaluation for plugin: %s" % str(inst)
logger.exception(msg)
return ServiceError(500, msg), 500
except Exception as ex:
msg = "Exception while performing evaluation for plugin: %s" % str(ex)
logger.exception(msg)
return ServiceError(500, msg), 500
self.add_task(task)
def _do_evaluate(self, assessment_type, metrics, input_data_path,
evaluation_input_format, ground_truth_path,
evaluation_path, properties):
"""
Performs metrics' evaluation using the predictions and ground truth files provided.
Stored the assessment results as a JSON file in the evaluation_path
:param assessment_type: The evaluation type. One of {'BinaryClassification',
'MultilabelClassification', 'MulticlassClassification', 'Regression'}
:param metrics: Specific metrics to evaluate against instead of all metrics defined by assessment_type
:param input_data_path: Path to input data for the evaluation
:param evaluation_input_format: The format of the input data
:param ground_truth_path: The directory path where the ground_truth.csv file is located
:param evaluation_path: A directory path to where the evaluation.json output file will be stored
:param properties: A dictionary of key value pairs for evaluation plugin arguments.
"""
logger.debug("_do_evaluation started")
try:
logger.info("Calling do_evaluation method.")
self.do_evaluate(assessment_type, metrics, input_data_path,
evaluation_input_format, ground_truth_path,
evaluation_path, properties)
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_evaluation")
self.fail(str(ex))
logger.debug("_do_evaluation complete")
def _do_resume_training(self):
"""
Unpauses the current activity (training or prediction).
"""
try:
self.do_resume_training()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_resume_training")
self.fail(str(ex))
def _do_resume_predict(self):
"""
Executes/resumes the prediction activity
"""
try:
self.do_resume_predict()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_resume_predict")
self.fail(str(ex))
def _do_generate(self):
"""
Executes/resumes the generation activity
"""
try:
self.do_generate()
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_generate")
self.fail(str(ex))
def _do_stream_predict(self, data_map: dict):
"""
Executes/resumes the stream prediction activity
"""
try:
response = self.do_stream_predict(data_map)
self.endpoint_service.put_response(response)
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_stream_predict")
self.fail(str(ex))
def _do_terminate(self):
"""
Stops all processing and releases any resources that are in use in
preparation for being shut down.
"""
try:
self.do_terminate()
self.terminated()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_terminate")
self.fail(str(ex))
def terminate(self):
"""
Shuts down the model and the endpoint service
"""
logger.debug("Terminate called")
try:
self.add_task(ModelInstanceTask(operation="terminate"))
except RuntimeError as inst:
msg = "Error while terminating the model: %s" % str(inst)
logger.exception(msg)
return ServiceError(500, msg), 500
def _do_reset(self):
"""
Resets the model into its initial state
"""
try:
self.do_reset()
self._model_built = False
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_reset")
self.fail(str(ex))
def _do_update_stream_properties(self, props: dict):
"""
Updates the stream prediction properties
:param props: A dictionary of streaming properties
"""
try:
self.do_update_stream_properties(props)
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_update_stream_properties")
self.fail(str(ex))
def _do_transform(self, inputDirs, outputDir, properties):
"""
Executes the transform activity
"""
logger.debug("_do_transform started")
try:
logger.info("Calling do_transform method.")
self.do_transform(inputDirs, outputDir, properties)
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_transform")
self.fail(str(ex))
logger.debug("_do_transform complete")
def predict(self):
"""
Creates and returns a Task which kicks off a prediction activity
:return: The created Task object
"""
logger.debug("Predict called")
try:
self.add_task(ModelInstanceTask(operation="predict"))
except RuntimeError as inst:
msg = "Error while kicking off prediction activity: %s" % str(inst)
logger.exception(msg)
return ServiceError(500, msg), 500
def _do_train(self):
"""
Executes/resumes the training activity
"""
logger.debug("_do_train started")
try:
logger.info("Calling do_train method.")
self.do_train()
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_train")
self.fail(str(ex))
logger.debug("_do_train complete")
def metrics(self):
"""
Metrics that can be performed by the evaluate method
"""
logger.debug("metrics started")
try:
metrics_list = self.plugin_manager.get_metrics_list()
logger.debug("metrics complete")
except Exception as ex:
logger.exception("Error running metrics")
self.fail(str(ex))
return metrics_list
def _do_save_predictions(self, dataPath):
"""
Saves the current prediction to the location specified
:param dataPath: The location on the local file system or distributed
file system where the predictions will be saved
"""
try:
self.do_save_predictions(dataPath)
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_save_predictions")
self.fail(str(ex))
def _do_build_model(self, path=None):
"""
Instructs the service to build all necessary data structures given the architecture and selected hyperparameters.
:param path: The path to the model file
"""
try:
self.do_build_model(path)
self._model_built = True
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_build_model")
self.fail(str(ex))
def get_api_version(self):
"""
Returns the version of the MISTK API
:return: The MISTK API version as text
"""
try:
version = pkg_resources.require("mistk")[0].version
return version, 200
except Exception as ex:
msg = 'Error occurred while attempting to retrieve MISTK API version: %s' % str(ex)
logger.exception(msg)
return ServiceError(500, msg), 500
def terminate(self): # noqa: E501
"""
Shutdowns the transform plugin and cleans up any resources.
:rtype: None
"""
logger.debug("Terminate called")
try:
self.add_task(TransformPluginTask(operation="terminate"))
except RuntimeError as inst:
msg = "Error while terminating the transform plugin and cleaning up resources. %s" % str(
inst)
logger.exception(msg)
return ServiceError(500, msg), 500
def _do_save_model(self, path):
"""
Save the model to an agreed upon location. It will be up to some other process to make sure
the saved model ends up back in the repository. This call could be a NOOP if, for instance,
the model is saved periodically throughout the training process.
:param path: The path to which the model should be saved.
"""
try:
self.do_save_model(path)
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_save_model")
self.fail(str(ex))
def update_stream_properties(self, props):
"""
Creates a task for updating the streaming prediction properties.
:param props: Dictionary of metadata properties to be used by the model
"""
try:
task = ModelInstanceTask(operation="update_stream_properties",
parameters={"props": props})
self.add_task(task)
except RuntimeError as inst:
msg = "Error while kicking off stream prediction activity: %s" % str(inst)
logger.exception(msg)
return ServiceError(500, msg), 500
def save_model(self, modelPath):
"""
Creates and returns a Task which saves the model to the path provided
:param modelPath: The path where the model will be saved.
:return: The created Task object
"""
try:
task = ModelInstanceTask(operation="save_model",
parameters = {"path": modelPath})
self.add_task(task)
except RuntimeError as inst:
msg = "Error while saving model to path provided: %s" % str(inst)
logger.exception(msg)
return ServiceError(500, msg), 500
def build_model(self, modelPath=None):
"""
Creates and returns a Task which builds the model using the modelPath provided
:param modelPath: The path to where the model image can be found
:return: The created Task object
"""
logger.debug("build model called")
try:
task = ModelInstanceTask(operation="build_model",
parameters = {"path": modelPath})
self.add_task(task)
except RuntimeError as inst:
msg = "Error while building model: %s" % str(inst)
logger.exception(msg)
return ServiceError(500, msg), 500
def assessment_types(self):
"""
Assessment types that are supported by the evaluate method
"""
types = []
try:
metrics_list = self.plugin_manager.get_metrics_list()
for metric in metrics_list:
for assessment_types in metric.assessment_types():
if type not in assessment_types:
types.append(type)
except Exception as ex:
logger.exception("Error running metrics")
self.fail(str(ex))
return types
def _do_load_data(self, dataset_map: dict):
"""
Instructs the container to load data (or at least record in memory where the data is, if it’s actually to be loaded during training).
:param dataset_map: A dictionary that maps string keys {training_data, test_data} to a
Dataset object that contains information on the dataset to load
"""
try:
self.do_load_data(dataset_map)
if self._model_built:
self.ready()
else:
self.initialized()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_load_data")
self.fail(str(ex))
def reset(self):
"""
Resets the model
"""
logger.debug("Reset called")
try:
def generate():
try:
yield "resetting..."
finally:
os.execv(sys.executable, ['python'] + sys.argv)
return Response(generate(), mimetype='text/plain')
except RuntimeError as inst:
msg = "Error while resetting the model: %s" % str(inst)
logger.exception(msg)
return ServiceError(500, msg), 500
