def report_failure(self, reason): """ Reports the failure of a model during its operations :param reason: The error message explaining why the model failed. """ logger.info('Model has failed: ' + str(reason))
def _do_evaluate(self, assessment_type, metrics, input_data_path, evaluation_input_format, ground_truth_path, evaluation_path, properties): """ Performs metrics' evaluation using the predictions and ground truth files provided. Stored the assessment results as a JSON file in the evaluation_path :param assessment_type: The evaluation type. One of {'BinaryClassification', 'MultilabelClassification', 'MulticlassClassification', 'Regression'} :param metrics: Specific metrics to evaluate against instead of all metrics defined by assessment_type :param input_data_path: Path to input data for the evaluation :param evaluation_input_format: The format of the input data :param ground_truth_path: The directory path where the ground_truth.csv file is located :param evaluation_path: A directory path to where the evaluation.json output file will be stored :param properties: A dictionary of key value pairs for evaluation plugin arguments. """ logger.debug("_do_evaluation started") try: logger.info("Calling do_evaluation method.") self.do_evaluate(assessment_type, metrics, input_data_path, evaluation_input_format, ground_truth_path, evaluation_path, properties) self.ready() except Exception as ex: #pylint: disable=broad-except logger.exception("Error running do_evaluation") self.fail(str(ex)) logger.debug("_do_evaluation complete")
def __init__(self): """ Initializes the Model Instance Endpoint service """ initializeEndpointController(self, transform_plugin_endpoint_controller) self.app = cx.FlaskApp('mistk.transform.server') self.app.app.json_encoder = mistk.data.utils.PresumptiveJSONEncoder self.app.add_api(self._load_api_spec()) self.http_server = None self._state_machine = None self._transform_plugin = None self._current_task = None self._status_lock = RWLock() self._task_lock = RWLock() self._old_tasks = list() self._thread_pool = ThreadPoolExecutor() info = ObjectInfo('TransformInstanceStatus', resource_version=1) self._status = TransformInstanceStatus(object_info=info, state='started') logger.info('Transform Plugin initialized')
def _do_train(self): """ Executes/resumes the training activity """ logger.debug("_do_train started") try: logger.info("Calling do_train method.") self.do_train() self.ready() except Exception as ex: #pylint: disable=broad-except logger.exception("Error running do_train") self.fail(str(ex)) logger.debug("_do_train complete")
def new_state_entered(self, *args, **kwargs): """ Notifies the endpoint service that the current state of the state machine has been update :param args: Optional non-keyworded variable length arguments to pass in :param kwargs: Optional keyworded variable length arguments to pass in """ logger.info("New state entered - %s {args: %s, kwargs: %s}", self.state, args, kwargs) if self.state == 'failed' and len(args) > 0: self.endpoint_service.update_state(self.state, payload=args[0]) else: self.endpoint_service.update_state(self.state)
def _do_transform(self, inputDirs, outputDir, properties): """ Executes the transform activity """ logger.debug("_do_transform started") try: logger.info("Calling do_transform method.") self.do_transform(inputDirs, outputDir, properties) self.ready() except Exception as ex: #pylint: disable=broad-except logger.exception("Error running do_transform") self.fail(str(ex)) logger.debug("_do_transform complete")
def _process_task(self): """ Processes the currently queued Task and updates its status as appropriate """ try: logger.info('Processing task %s', self._current_task.operation) m = getattr(self.model, self._current_task.operation) m(**(self._current_task.parameters or {})) with self._task_lock.writer_lock: self._current_task.status = 'complete' self._current_task.completed = datetime.now() logger.info('Processing of task is complete') except Exception as ex: #pylint: disable=broad-except logger.exception("Error occurred running task") self._current_task.status = 'failed' self._current_task.message = str(ex) raise
def validate_groundtruth_csv(file_path): """ Validates a ground truth csv file :param path: The directory or file path where the ground truth csv file can be found :returns: True if the csv file is valid, false otherwise. """ logger.info("Validating Ground Truth CSV file at %s" % file_path) csv_file = '' if os.path.isfile(file_path): csv_file = file_path elif os.path.isdir(file_path) and os.path.isfile( os.path.join(file_path, "ground_truth.csv")): csv_file = os.path.join(file_path, "ground_truth.csv") else: logger.error("No groundtruth file exists at %s" % file_path) return False return _validate_csv(csv_file)
def validate_predictions_csv(file_path): """ Validates a predictions csv file :param path: The directory or file path where the predictions csv file can be found :returns: True if the csv file is valid, false otherwise. """ logger.info("Validating Predictions CSV file at %s" % file_path) csv_file = '' if os.path.isfile(file_path): csv_file = file_path elif os.path.isdir(file_path) and os.path.isfile( os.path.join(file_path, "predictions.csv")): csv_file = os.path.join(file_path, "predictions.csv") else: logger.error("No predictions file exists at %s" % file_path) return False return _validate_csv(csv_file)
def _read_metrics(self, uri): if not os.path.exists(uri): logger.warn( "Plugin defaults file does not exist at %s." "It will now be created and populated with" " default values", uri) os.makedirs(os.path.dirname(uri), exist_ok=True) src = os.path.join(os.path.dirname(__file__), "metrics.json") shutil.copy(src, uri) with self._read(uri) as reader: metric_dict_list = json.load(reader) self._default_metrics = metric_dict_list for metric_dict in metric_dict_list: logger.info('metric json loading: ' + str(metric_dict)) metric_object = datautils.deserialize_model( metric_dict, MistkMetric) logger.info('metric loaded: ' + str(metric_object)) if metric_object.package and metric_object.method: self._metric_dict[metric_object.package + '.' + metric_object.method] = metric_object else: self._metric_dict[ metric_object.object_info.name] = metric_object self._metric_list.append(metric_object) logger.info('Metrics loaded.')
def terminated(self): """ Terminates the model """ logger.info("Shutting down") sys.exit()
def do_evaluate(self, assessment_type, metrics, input_data_path, evaluation_input_format, ground_truth_path, evaluation_path, properties): """ Performs metrics' evaluation using the predictions and ground truth files provided. Stored the assessment results as a JSON file in the evaluation_path :param assessment_type: The evaluation assessment type. One of {'BinaryClassification', 'MultilabelClassification', 'MulticlassClassification', 'Regression'} :param metrics: Specific metrics to evaluate against instead of all metrics defined by assessment_type :param input_data_path: Path to input data for the evaluation :param evaluation_input_format: The format of the input data :param ground_truth_path: The directory path where the ground_truth.csv file is located :param evaluation_path: A directory path to where the evaluation.json output file will be stored :param properties: A dictionary of key value pairs for evaluation plugin arguments. """ if evaluation_input_format not in "predictions": msg = "EvaluationInputFormat %s is not supported by this Metric Evaluator, only 'predictions' are supported" % evaluation_input_format logger.error(msg) raise Exception(msg) # load prediction results full_predictions_path = os.path.join(input_data_path, "predictions.csv") results_df = csv_Predictions_to_DataFrame(full_predictions_path) # load ground truth full_ground_truth_path = os.path.join(ground_truth_path, "ground_truth.csv") truth_df = csv_Groundtruth_to_DataFrame(full_ground_truth_path) # match ground truth to results by id truth_df = truth_df.loc[truth_df['rowid'].isin(results_df['rowid'])] # sort the rows by id results_df.sort_values(by='rowid', inplace=True) truth_df.sort_values(by='rowid', inplace=True) logger.debug('Running for metrics %s' % metrics) if assessment_type == "MultilabelClassification" or assessment_type == "MulticlassClassification": # create matrices for labels and confidence label_mlb = MultiLabelBinarizer() parsed_truth_labels = ( truth_df['labels'].str.split().values.tolist() if truth_df['labels'].dtype == 'object' else np.array( np.transpose(np.matrix(truth_df['labels'].values)))) parsed_results_labels = ( results_df['labels'].str.split().values.tolist() if results_df['labels'].dtype == 'object' else np.array( np.transpose(np.matrix(results_df['labels'].values)))) label_mlb.fit( np.append(parsed_truth_labels, parsed_results_labels, axis=0)) truth_labels_matrix = label_mlb.transform(parsed_truth_labels) results_labels_matrix = label_mlb.transform(parsed_results_labels) if 'confidence' in results_df and not results_df[ 'confidence'].hasnans: parsed_confidence = ( results_df['confidence'].str.split().values.tolist() if results_df['confidence'].dtype == 'object' else np.array( np.transpose(np.matrix( results_df['confidence'].values)))) confidence_matrix = np.empty(results_labels_matrix.shape) label_classes = label_mlb.classes_.tolist() for row_index, row in enumerate(parsed_results_labels): confidence_row = np.zeros(results_labels_matrix.shape[1]) for col_index, col in enumerate(row): label_pos = label_classes.index(col) confidence_row[label_pos] = np.float64( parsed_confidence[row_index][col_index]) #pylint: disable=no-member confidence_matrix[row_index] = confidence_row elif assessment_type == "Regression": if truth_df['labels'].dtype == 'object': truth_labels_matrix = truth_df['labels'].str.split( ).values.tolist() for index, item in enumerate(truth_labels_matrix): truth_labels_matrix[index] = np.array(item, dtype=np.float64) #pylint: disable=no-member else: truth_labels_matrix = truth_df['labels'].values if results_df['labels'].dtype == 'object': results_labels_matrix = results_df['labels'].str.split( ).values.tolist() for index, item in enumerate(results_labels_matrix): results_labels_matrix[index] = np.array(item, dtype=np.float64) #pylint: disable=no-member else: results_labels_matrix = results_df['labels'].values if results_df['confidence'].dtype == 'object': confidence_matrix = results_df['confidence'].str.split( ).values.tolist() for index, item in enumerate(confidence_matrix): confidence_matrix[index] = np.array(item, dtype=np.float64) #pylint: disable=no-member else: confidence_matrix = results_df['confidence'].values else: truth_labels_matrix = ( truth_df['labels'].str.split().values.tolist() if truth_df['labels'].dtype == 'object' else truth_df['labels'].values) results_labels_matrix = ( results_df['labels'].str.split().values.tolist() if results_df['labels'].dtype == 'object' else results_df['labels'].values) confidence_matrix = ( results_df['confidence'].str.split().values.tolist() if results_df['confidence'].dtype == 'object' else results_df['confidence'].values) eval_dict = {} modules_cache = {} for counter, metric in enumerate(metrics): logger.info(metric.package + " : " + metric.method) if metric.package not in modules_cache: module = None name = metric.package try: importlib.invalidate_caches() module = importlib.import_module(name) except Exception: logger.exception("Exception importing plugin module " + name) if module: modules_cache[metric.package] = module else: logger.warn("Cannot load " + metric.package) continue else: logger.debug("Loading cached module") module = modules_cache[metric.package] if hasattr(module, metric.method): logger.debug("Calling " + metric.method + " in " + metric.package) method = getattr(module, metric.method) args = metric.default_args or {} if metric.data_parameters.truth_labels: args[metric.data_parameters. truth_labels] = truth_labels_matrix if metric.data_parameters.truth_bounds and not truth_df[ 'bounds'].hasnans: args[metric.data_parameters. truth_bounds] = truth_df['bounds'].values if metric.data_parameters.prediction_labels: args[metric.data_parameters. prediction_labels] = results_labels_matrix if metric.data_parameters.prediction_scores and 'confidence' in results_df and not results_df[ 'confidence'].hasnans: args[metric.data_parameters. prediction_scores] = confidence_matrix if metric.data_parameters.prediction_bounds and not results_df[ 'bounds'].hasnans: args[metric.data_parameters. prediction_bounds] = results_df['bounds'].values try: evalResult = method(**args) except Exception: logger.error("Something bad happened calling " + metric.method, exc_info=True) else: logger.debug("Result is " + str(evalResult)) if isinstance(evalResult, np.ndarray): # convert to native types evalResultAsList = evalResult.tolist() if assessment_type == "MultilabelClassification" or assessment_type == "MulticlassClassification": # map labels to their values in the results label_classes = label_mlb.classes_.tolist() if len(evalResultAsList) == len(label_classes): evalResultAsDict = {} for index, label in enumerate(label_classes): evalResultAsDict[str( label)] = evalResultAsList[index] eval_dict[ metric.object_info.name] = evalResultAsDict else: eval_dict[ metric.object_info.name] = evalResultAsList else: eval_dict[ metric.object_info.name] = evalResultAsList elif isinstance(evalResult, np.generic): # convert to native type evalResultAsScalar = np.asscalar(evalResult) eval_dict[metric.object_info.name] = evalResultAsScalar elif isinstance(evalResult, tuple) or isinstance( evalResult, list): # kind of a cheat to cover the case where a native type has numpy elements # which some scikit-learn methods inexplicably return eval_dict[metric.object_info.name] = np.array( evalResult).tolist() else: eval_dict[metric.object_info.name] = evalResult else: logger.warn(metric.method + " does not exist in " + metric.package) logger.info("Completed metric " + str(counter + 1)) eval_dict_json = json.dumps(eval_dict, indent=2) filename = evaluation_path + "/eval_results_" + str(int( time.time())) + ".json" logger.info("Writing eval results to " + filename) with open(filename, mode='w') as writer: writer.write(eval_dict_json)