def report_failure(self, reason):
"""
Reports the failure of a model during its operations
:param reason: The error message explaining why the model failed.
"""
logger.info('Model has failed: ' + str(reason))
def _do_evaluate(self, assessment_type, metrics, input_data_path,
evaluation_input_format, ground_truth_path,
evaluation_path, properties):
"""
Performs metrics' evaluation using the predictions and ground truth files provided.
Stored the assessment results as a JSON file in the evaluation_path
:param assessment_type: The evaluation type. One of {'BinaryClassification',
'MultilabelClassification', 'MulticlassClassification', 'Regression'}
:param metrics: Specific metrics to evaluate against instead of all metrics defined by assessment_type
:param input_data_path: Path to input data for the evaluation
:param evaluation_input_format: The format of the input data
:param ground_truth_path: The directory path where the ground_truth.csv file is located
:param evaluation_path: A directory path to where the evaluation.json output file will be stored
:param properties: A dictionary of key value pairs for evaluation plugin arguments.
"""
logger.debug("_do_evaluation started")
try:
logger.info("Calling do_evaluation method.")
self.do_evaluate(assessment_type, metrics, input_data_path,
evaluation_input_format, ground_truth_path,
evaluation_path, properties)
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_evaluation")
self.fail(str(ex))
logger.debug("_do_evaluation complete")
def __init__(self):
"""
Initializes the Model Instance Endpoint service
"""
initializeEndpointController(self,
transform_plugin_endpoint_controller)
self.app = cx.FlaskApp('mistk.transform.server')
self.app.app.json_encoder = mistk.data.utils.PresumptiveJSONEncoder
self.app.add_api(self._load_api_spec())
self.http_server = None
self._state_machine = None
self._transform_plugin = None
self._current_task = None
self._status_lock = RWLock()
self._task_lock = RWLock()
self._old_tasks = list()
self._thread_pool = ThreadPoolExecutor()
info = ObjectInfo('TransformInstanceStatus', resource_version=1)
self._status = TransformInstanceStatus(object_info=info,
state='started')
logger.info('Transform Plugin initialized')
def _do_train(self):
"""
Executes/resumes the training activity
"""
logger.debug("_do_train started")
try:
logger.info("Calling do_train method.")
self.do_train()
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_train")
self.fail(str(ex))
logger.debug("_do_train complete")
def new_state_entered(self, *args, **kwargs):
"""
Notifies the endpoint service that the current state of the state machine has been update
:param args: Optional non-keyworded variable length arguments to pass in
:param kwargs: Optional keyworded variable length arguments to pass in
"""
logger.info("New state entered - %s {args: %s, kwargs: %s}",
self.state, args, kwargs)
if self.state == 'failed' and len(args) > 0:
self.endpoint_service.update_state(self.state, payload=args[0])
else:
self.endpoint_service.update_state(self.state)
def _do_transform(self, inputDirs, outputDir, properties):
"""
Executes the transform activity
"""
logger.debug("_do_transform started")
try:
logger.info("Calling do_transform method.")
self.do_transform(inputDirs, outputDir, properties)
self.ready()
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error running do_transform")
self.fail(str(ex))
logger.debug("_do_transform complete")
def _process_task(self):
"""
Processes the currently queued Task and updates its status as appropriate
"""
try:
logger.info('Processing task %s', self._current_task.operation)
m = getattr(self.model, self._current_task.operation)
m(**(self._current_task.parameters or {}))
with self._task_lock.writer_lock:
self._current_task.status = 'complete'
self._current_task.completed = datetime.now()
logger.info('Processing of task is complete')
except Exception as ex: #pylint: disable=broad-except
logger.exception("Error occurred running task")
self._current_task.status = 'failed'
self._current_task.message = str(ex)
raise
def validate_groundtruth_csv(file_path):
"""
Validates a ground truth csv file
:param path: The directory or file path where the ground truth
csv file can be found
:returns: True if the csv file is valid, false otherwise.
"""
logger.info("Validating Ground Truth CSV file at %s" % file_path)
csv_file = ''
if os.path.isfile(file_path):
csv_file = file_path
elif os.path.isdir(file_path) and os.path.isfile(
os.path.join(file_path, "ground_truth.csv")):
csv_file = os.path.join(file_path, "ground_truth.csv")
else:
logger.error("No groundtruth file exists at %s" % file_path)
return False
return _validate_csv(csv_file)
def validate_predictions_csv(file_path):
"""
Validates a predictions csv file
:param path: The directory or file path where the predictions
csv file can be found
:returns: True if the csv file is valid, false otherwise.
"""
logger.info("Validating Predictions CSV file at %s" % file_path)
csv_file = ''
if os.path.isfile(file_path):
csv_file = file_path
elif os.path.isdir(file_path) and os.path.isfile(
os.path.join(file_path, "predictions.csv")):
csv_file = os.path.join(file_path, "predictions.csv")
else:
logger.error("No predictions file exists at %s" % file_path)
return False
return _validate_csv(csv_file)
def _read_metrics(self, uri):
if not os.path.exists(uri):
logger.warn(
"Plugin defaults file does not exist at %s."
"It will now be created and populated with"
" default values", uri)
os.makedirs(os.path.dirname(uri), exist_ok=True)
src = os.path.join(os.path.dirname(__file__), "metrics.json")
shutil.copy(src, uri)
with self._read(uri) as reader:
metric_dict_list = json.load(reader)
self._default_metrics = metric_dict_list
for metric_dict in metric_dict_list:
logger.info('metric json loading: ' + str(metric_dict))
metric_object = datautils.deserialize_model(
metric_dict, MistkMetric)
logger.info('metric loaded: ' + str(metric_object))
if metric_object.package and metric_object.method:
self._metric_dict[metric_object.package + '.' +
metric_object.method] = metric_object
else:
self._metric_dict[
metric_object.object_info.name] = metric_object
self._metric_list.append(metric_object)
logger.info('Metrics loaded.')
def terminated(self):
"""
Terminates the model
"""
logger.info("Shutting down")
sys.exit()
def do_evaluate(self, assessment_type, metrics, input_data_path,
evaluation_input_format, ground_truth_path,
evaluation_path, properties):
"""
Performs metrics' evaluation using the predictions and ground truth files provided.
Stored the assessment results as a JSON file in the evaluation_path
:param assessment_type: The evaluation assessment type. One of {'BinaryClassification',
'MultilabelClassification', 'MulticlassClassification', 'Regression'}
:param metrics: Specific metrics to evaluate against instead of all metrics defined by assessment_type
:param input_data_path: Path to input data for the evaluation
:param evaluation_input_format: The format of the input data
:param ground_truth_path: The directory path where the ground_truth.csv file is located
:param evaluation_path: A directory path to where the evaluation.json output file will be stored
:param properties: A dictionary of key value pairs for evaluation plugin arguments.
"""
if evaluation_input_format not in "predictions":
msg = "EvaluationInputFormat %s is not supported by this Metric Evaluator, only 'predictions' are supported" % evaluation_input_format
logger.error(msg)
raise Exception(msg)
# load prediction results
full_predictions_path = os.path.join(input_data_path,
"predictions.csv")
results_df = csv_Predictions_to_DataFrame(full_predictions_path)
# load ground truth
full_ground_truth_path = os.path.join(ground_truth_path,
"ground_truth.csv")
truth_df = csv_Groundtruth_to_DataFrame(full_ground_truth_path)
# match ground truth to results by id
truth_df = truth_df.loc[truth_df['rowid'].isin(results_df['rowid'])]
# sort the rows by id
results_df.sort_values(by='rowid', inplace=True)
truth_df.sort_values(by='rowid', inplace=True)
logger.debug('Running for metrics %s' % metrics)
if assessment_type == "MultilabelClassification" or assessment_type == "MulticlassClassification":
# create matrices for labels and confidence
label_mlb = MultiLabelBinarizer()
parsed_truth_labels = (
truth_df['labels'].str.split().values.tolist()
if truth_df['labels'].dtype == 'object' else np.array(
np.transpose(np.matrix(truth_df['labels'].values))))
parsed_results_labels = (
results_df['labels'].str.split().values.tolist()
if results_df['labels'].dtype == 'object' else np.array(
np.transpose(np.matrix(results_df['labels'].values))))
label_mlb.fit(
np.append(parsed_truth_labels, parsed_results_labels, axis=0))
truth_labels_matrix = label_mlb.transform(parsed_truth_labels)
results_labels_matrix = label_mlb.transform(parsed_results_labels)
if 'confidence' in results_df and not results_df[
'confidence'].hasnans:
parsed_confidence = (
results_df['confidence'].str.split().values.tolist() if
results_df['confidence'].dtype == 'object' else np.array(
np.transpose(np.matrix(
results_df['confidence'].values))))
confidence_matrix = np.empty(results_labels_matrix.shape)
label_classes = label_mlb.classes_.tolist()
for row_index, row in enumerate(parsed_results_labels):
confidence_row = np.zeros(results_labels_matrix.shape[1])
for col_index, col in enumerate(row):
label_pos = label_classes.index(col)
confidence_row[label_pos] = np.float64(
parsed_confidence[row_index][col_index]) #pylint: disable=no-member
confidence_matrix[row_index] = confidence_row
elif assessment_type == "Regression":
if truth_df['labels'].dtype == 'object':
truth_labels_matrix = truth_df['labels'].str.split(
).values.tolist()
for index, item in enumerate(truth_labels_matrix):
truth_labels_matrix[index] = np.array(item,
dtype=np.float64) #pylint: disable=no-member
else:
truth_labels_matrix = truth_df['labels'].values
if results_df['labels'].dtype == 'object':
results_labels_matrix = results_df['labels'].str.split(
).values.tolist()
for index, item in enumerate(results_labels_matrix):
results_labels_matrix[index] = np.array(item,
dtype=np.float64) #pylint: disable=no-member
else:
results_labels_matrix = results_df['labels'].values
if results_df['confidence'].dtype == 'object':
confidence_matrix = results_df['confidence'].str.split(
).values.tolist()
for index, item in enumerate(confidence_matrix):
confidence_matrix[index] = np.array(item, dtype=np.float64) #pylint: disable=no-member
else:
confidence_matrix = results_df['confidence'].values
else:
truth_labels_matrix = (
truth_df['labels'].str.split().values.tolist()
if truth_df['labels'].dtype == 'object' else
truth_df['labels'].values)
results_labels_matrix = (
results_df['labels'].str.split().values.tolist()
if results_df['labels'].dtype == 'object' else
results_df['labels'].values)
confidence_matrix = (
results_df['confidence'].str.split().values.tolist()
if results_df['confidence'].dtype == 'object' else
results_df['confidence'].values)
eval_dict = {}
modules_cache = {}
for counter, metric in enumerate(metrics):
logger.info(metric.package + " : " + metric.method)
if metric.package not in modules_cache:
module = None
name = metric.package
try:
importlib.invalidate_caches()
module = importlib.import_module(name)
except Exception:
logger.exception("Exception importing plugin module " +
name)
if module:
modules_cache[metric.package] = module
else:
logger.warn("Cannot load " + metric.package)
continue
else:
logger.debug("Loading cached module")
module = modules_cache[metric.package]
if hasattr(module, metric.method):
logger.debug("Calling " + metric.method + " in " +
metric.package)
method = getattr(module, metric.method)
args = metric.default_args or {}
if metric.data_parameters.truth_labels:
args[metric.data_parameters.
truth_labels] = truth_labels_matrix
if metric.data_parameters.truth_bounds and not truth_df[
'bounds'].hasnans:
args[metric.data_parameters.
truth_bounds] = truth_df['bounds'].values
if metric.data_parameters.prediction_labels:
args[metric.data_parameters.
prediction_labels] = results_labels_matrix
if metric.data_parameters.prediction_scores and 'confidence' in results_df and not results_df[
'confidence'].hasnans:
args[metric.data_parameters.
prediction_scores] = confidence_matrix
if metric.data_parameters.prediction_bounds and not results_df[
'bounds'].hasnans:
args[metric.data_parameters.
prediction_bounds] = results_df['bounds'].values
try:
evalResult = method(**args)
except Exception:
logger.error("Something bad happened calling " +
metric.method,
exc_info=True)
else:
logger.debug("Result is " + str(evalResult))
if isinstance(evalResult, np.ndarray):
# convert to native types
evalResultAsList = evalResult.tolist()
if assessment_type == "MultilabelClassification" or assessment_type == "MulticlassClassification":
# map labels to their values in the results
label_classes = label_mlb.classes_.tolist()
if len(evalResultAsList) == len(label_classes):
evalResultAsDict = {}
for index, label in enumerate(label_classes):
evalResultAsDict[str(
label)] = evalResultAsList[index]
eval_dict[
metric.object_info.name] = evalResultAsDict
else:
eval_dict[
metric.object_info.name] = evalResultAsList
else:
eval_dict[
metric.object_info.name] = evalResultAsList
elif isinstance(evalResult, np.generic):
# convert to native type
evalResultAsScalar = np.asscalar(evalResult)
eval_dict[metric.object_info.name] = evalResultAsScalar
elif isinstance(evalResult, tuple) or isinstance(
evalResult, list):
# kind of a cheat to cover the case where a native type has numpy elements
# which some scikit-learn methods inexplicably return
eval_dict[metric.object_info.name] = np.array(
evalResult).tolist()
else:
eval_dict[metric.object_info.name] = evalResult
else:
logger.warn(metric.method + " does not exist in " +
metric.package)
logger.info("Completed metric " + str(counter + 1))
eval_dict_json = json.dumps(eval_dict, indent=2)
filename = evaluation_path + "/eval_results_" + str(int(
time.time())) + ".json"
logger.info("Writing eval results to " + filename)
with open(filename, mode='w') as writer:
writer.write(eval_dict_json)
