def main(): options = parse_args() sc = SparkContext(appName="predict-test") pm.init(sc) predict_node(options) sc.stop() pm.done()
def db_to_df(engine, table): """ Save DataFrame to Database """ mlops.init() df_sink = pandas.read_sql("{} {}".format(SELECT_STATEMENT, table), con = engine) mlops.set_stat(table, df_sink.shape[0]) mlops.done() return(df_sink, df_sink.shape[0])
def _materialize(self, parent_data_objs, user_data): # Initialize MLOps Library mlops.init() df_data = parent_data_objs[0] df_clean = do_nan_removal(df_data) get_data_distribution_stat(df_clean) # Terminate MLOPs mlops.done() return [df_clean]
def test_set_stat_basic(): with pytest.raises(MLOpsException): pm.set_stat(name=None, data=None) pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) pm.set_stat("st1", data=5.5, category=StatCategory.TIME_SERIES) pm.set_stat("st1", data=5.5) pm.done()
def test_mlops_structure_api(): ion_instance_id = ION1.ION_INSTANCE_ID ion_node_id = ION1.NODE_1_ID token = ION1.TOKEN set_mlops_env(ion_id=ion_instance_id, ion_node_id=ion_node_id, token=token, model_id=ION1.MODEL_ID) rest_helper = MlOpsRestFactory().get_rest_helper(MLOpsMode.AGENT, mlops_server="localhost", mlops_port="3456", token=token) rest_helper.set_prefix(Constants.URL_MLOPS_PREFIX) with requests_mock.mock() as m: m.get(rest_helper.url_get_workflow_instance(ion_instance_id), json=test_workflow_instances) m.get(rest_helper.url_get_ees(), json=test_ee_info) m.get(rest_helper.url_get_agents(), json=test_agents_info) m.get(rest_helper.url_get_model_list(), json=test_models_info) m.get(rest_helper.url_get_health_thresholds(ion_instance_id), json=test_health_info) m.get(rest_helper.url_get_model_stats(ION1.MODEL_ID), json=test_model_stats) m.get(rest_helper.url_get_uuid("model"), json={"id": "model_5906255e-0a3d-4fef-8653-8d41911264fb"}) pm.init(ctx=None, mlops_mode=MLOpsMode.AGENT) assert pm.get_mlapp_id() == ION1.ION_ID assert pm.get_mlapp_name() == ION1.ION_NAME curr_node = pm.get_current_node() assert curr_node.id == ion_node_id nodes = pm.get_nodes() assert len(nodes) == 2 node0 = pm.get_node('1') assert node0 is not None assert node0.pipeline_pattern_id == ION1.PIPELINE_PATTERN_ID_1 assert node0.pipeline_instance_id == ION1.PIPELINE_INST_ID_1 node0_agents = pm.get_agents('1') assert len(node0_agents) == 1 assert node0_agents[0].id == ION1.AGENT_ID_0 assert node0_agents[0].hostname == 'localhost' agent = pm.get_agent('1', ION1.AGENT_ID_0) assert agent.id == ION1.AGENT_ID_0 assert agent.hostname == 'localhost' model = pm.current_model() assert model is not None assert model.metadata.modelId == ION1.MODEL_ID pm.done()
def _cleanup_on_exist(self): if self._ml_engine: self._ml_engine.stop() if mlops_loaded and mlops.init_called: mlops.done() if self._ml_engine: self._ml_engine.cleanup()
def test_set_events_api(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) pm._set_api_test_mode() with pytest.raises(MLOpsException): pm.set_event(name=None, data=None, type=EventType.System) with pytest.raises(MLOpsException): pm.set_event(name="ssss", data=None, type=None) pm.done()
def test_data_distribution_stat_api(generate_da_with_missing_data): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) pm._set_api_test_mode() # basic test data = np.array([[1, 2], [3, 4]]) pm.set_data_distribution_stat(data) # test with column missing blah = pd.read_csv(generate_da_with_missing_data) pm.set_data_distribution_stat(blah) pm.done()
def test_data_distribution_stat_api_spark(spark_session, generate_da_with_missing_data): sc = spark_session.sparkContext pm.init(ctx=sc, mlops_mode=MLOpsMode.STAND_ALONE) pm._set_api_test_mode() pdf = pd.read_csv(generate_da_with_missing_data) spark_df = spark_session.createDataFrame(pdf) pm.set_data_distribution_stat(data=spark_df) sc.stop() pm.done()
def test_init_done(): """ Testing api for information such as ION id, ion name and such :return: """ with pytest.raises(MLOpsException): pm.set_stat("st1", 5.5) with pytest.raises(MLOpsException): pm.done() pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) pm.done()
def read_file_to_df(self, filepath): """ Read file and return DataFrame """ mlops.init() if not os.path.exists(filepath): self._logger.info("stderr- failed to find {}".format(filepath), file=sys.stderr) raise Exception("file path does not exist: {}".format(filepath)) test_data = pandas.read_csv(filepath) mlops.done() return test_data
def test_mlops_roc_curve_apis(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) labels_pred_prob = [0.9, 0.4, 0.6, 0.9, 0.1, 0.9] labels_actual = [0, 1, 0, 0, 0, 1] fpr, tpr, thresholds = sklearn.metrics.roc_curve(labels_actual, labels_pred_prob, pos_label=1, sample_weight=None, drop_intermediate=True) roc_auc_score = sklearn.metrics.roc_auc_score(labels_actual, labels_pred_prob) graph_label_str = "AUC: {}".format(roc_auc_score) # first way pm.set_stat(ClassificationMetrics.PRECISION_RECALL_CURVE, [tpr, fpr]) pm.set_stat(ClassificationMetrics.PRECISION_RECALL_CURVE, [tpr, fpr], legend=graph_label_str) # second way pm.metrics.roc_curve(labels_actual, labels_pred_prob, pos_label=1, sample_weight=None, drop_intermediate=True) # should throw error if labels predicted is different length than actuals with pytest.raises(ValueError): labels_prob_missing_values = [0.0, 0.9, 1.0, 0.85] pm.metrics.roc_curve(labels_actual, labels_prob_missing_values, pos_label=1, sample_weight=None, drop_intermediate=True) sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0] # testing with sample weights as well pm.metrics.roc_curve(labels_actual, labels_pred_prob, pos_label=1, sample_weight=sample_weight, drop_intermediate=True) pm.done()
def main(): options = parse_args() sc = SparkContext(appName="health-test") pm.init(sc) print("MLOps testing") print("My ION Node id: {}".format(mlops.get_current_node().id)) print("My ION Node name: {}".format(mlops.get_current_node().name)) run_mlops_tests(package_to_scan=parallelm.mlops.e2e_tests.health_node, test_to_run=options.test_name) sc.stop() pm.done()
def main(): sc = pyspark.SparkContext(appName="PySparkWordCount") mlops.init(sc) options = parse_args() if options.words_file is None: print("Bad option - no file was provided") mlops.set_stat("got_file", 0) else: mlops.set_stat("got_file", 1) count_words(sc, options.words_file) sc.stop() mlops.done()
def main(): sc = SparkContext(appName="fake ab") pm.init(sc) print("Test") try: pm.set_stat("samples", 10000, st.TIME_SERIES) pm.set_stat("conversions", randint(1, 10000), st.TIME_SERIES) except Exception as e: print("Got exception while getting stats: {}".format(e)) pm.set_stat("error", 1, st.TIME_SERIES) sc.stop() pm.done()
def test_mlops_confusion_metrics_apis(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) labels_pred = [1, 0, 1, 1, 1, 0] labels_actual = [0, 1, 0, 0, 0, 1] labels_ordered = [0, 1] cm = metrics.confusion_matrix(labels_actual, labels_pred, labels=labels_ordered) # first way pm.set_stat(ClassificationMetrics.CONFUSION_MATRIX, cm, labels=labels_ordered) # second way pm.metrics.confusion_matrix(y_true=labels_actual, y_pred=labels_pred, labels=labels_ordered) # should throw error if labels are not provided with pytest.raises(MLOpsStatisticsException): pm.set_stat(ClassificationMetrics.CONFUSION_MATRIX, cm) with pytest.raises(MLOpsStatisticsException): pm.metrics.confusion_matrix(y_true=labels_actual, y_pred=labels_pred, labels=None) # should throw error if labels predicted is different length than actuals with pytest.raises(ValueError): labels_pred_missing_values = [0, 0, 0, 1] pm.metrics.confusion_matrix(y_true=labels_actual, y_pred=labels_pred_missing_values, labels=None) sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0] # testing with sample weights as well pm.metrics.confusion_matrix(y_true=labels_actual, y_pred=labels_pred, labels=labels_ordered, sample_weight=sample_weight) pm.done()
def test_mlops_precision_recall_curve_apis(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) labels_pred_prob = [0.9, 0.4, 0.6, 0.9, 0.1, 0.9] labels_actual = [0, 1, 0, 0, 0, 1] precision, recall, thresholds = sklearn.metrics.precision_recall_curve( y_true=labels_actual, probas_pred=labels_pred_prob) # first way pm.set_stat(ClassificationMetrics.PRECISION_RECALL_CURVE, [precision, recall]) pm.set_stat(ClassificationMetrics.PRECISION_RECALL_CURVE, [precision, recall], legend="Precision Recall") # second way pm.metrics.precision_recall_curve(y_true=labels_actual, probas_pred=labels_pred_prob) # should throw error if labels predicted is different length than actuals with pytest.raises(ValueError): labels_prob_missing_values = [0.0, 0.9, 1.0, 0.85] pm.metrics.precision_recall_curve( y_true=labels_actual, probas_pred=labels_prob_missing_values) sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0] # testing with sample weights as well pm.metrics.precision_recall_curve(y_true=labels_actual, probas_pred=labels_pred_prob, sample_weight=sample_weight) # testing with pos label as well pm.metrics.precision_recall_curve(y_true=labels_actual, probas_pred=labels_pred_prob, sample_weight=sample_weight, pos_label=1) # testing with average as well pm.metrics.precision_recall_curve(y_true=labels_actual, probas_pred=labels_pred_prob, sample_weight=sample_weight, pos_label=1, average="micro") pm.done()
def test_mlops_mean_squared_log_error_apis(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) labels_pred = [1.0, 0.5, 2.5, 4.75, 7.0, 0.75] labels_actual = [1.5, 0.75, 2.75, 4.5, 7.50, 0.25] msle = sklearn.metrics.mean_squared_log_error(labels_actual, labels_pred) # first way pm.set_stat(RegressionMetrics.MEAN_SQUARED_LOG_ERROR, msle) pm.set_stat(RegressionMetrics.MEAN_SQUARED_LOG_ERROR, [1, 2, 3]) # second way pm.metrics.mean_squared_log_error(y_true=labels_actual, y_pred=labels_pred) # should throw error if labels predicted is different length than actuals with pytest.raises(ValueError): labels_pred_missing_values = [1.0, 0.5, 7.0, 0.75] pm.metrics.mean_squared_log_error(y_true=labels_actual, y_pred=labels_pred_missing_values) # should throw error if labels contain negative values with pytest.raises(ValueError): labels_pred_neg = [1.0, -0.5, 2.5, 4.75, 7.0, 0.75] labels_actual_neg = [1.5, -0.75, 2.75, 4.5, 7.50, 0.25] pm.metrics.mean_squared_log_error(y_true=labels_actual_neg, y_pred=labels_pred_neg) sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0] # testing with sample weights as well pm.metrics.mean_squared_log_error(y_true=labels_actual, y_pred=labels_pred, sample_weight=sample_weight) labels_2d_actual = [[1.0, 0.5], [2.5, 4.75], [7.0, 0.75]] labels_2d_pred = [[1.5, 0.75], [2.75, 4.5], [7.50, 0.25]] msle = pm.metrics.mean_squared_log_error(y_true=labels_2d_actual, y_pred=labels_2d_pred, multioutput="raw_values") assert len(msle) == 2 pm.done()
def main(): mlops.init() options = parse_args() for i in range(10): model = mlops.get_last_approved_model() if model: mlops.set_stat("model fetched", 10) mlops.set_stat("fetched model size", model.metadata.size) print("FETCHED MODEL: {}".format(model.metadata)) break else: mlops.set_stat("model fetched", 0) time.sleep(1) mlops.done()
def test_mlops_precision_score_apis(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) labels_pred = [1, 0, 1, 1, 1, 0] labels_actual = [0, 1, 0, 0, 0, 1] precision_score = sklearn.metrics.precision_score(labels_actual, labels_pred) # first way pm.set_stat(ClassificationMetrics.PRECISION_SCORE, precision_score) # second way pm.metrics.precision_score(y_true=labels_actual, y_pred=labels_pred) sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0] # testing with sample weights as well pm.metrics.precision_score(y_true=labels_actual, y_pred=labels_pred, sample_weight=sample_weight) labels_pred_multiclass = [1, 0, 2, 1, 1, 2] labels_actual_multiclass = [0, 1, 1, 0, 2, 0] # first way where precision score is array of values per class. precision_score = sklearn.metrics.precision_score( y_true=labels_actual_multiclass, y_pred=labels_pred_multiclass, pos_label=2, labels=[0, 1, 2], average=None) pm.set_stat(ClassificationMetrics.PRECISION_SCORE, precision_score) # second way precision_score = pm.metrics.precision_score( y_true=labels_actual_multiclass, y_pred=labels_pred_multiclass, pos_label=2, labels=[0, 1, 2], average=None) assert len(precision_score) == 3 pm.done()
def test_mlops_fbeta_score_apis(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) labels_pred = [1, 0, 1, 1, 1, 0] labels_actual = [0, 1, 0, 0, 0, 1] fbeta_score = sklearn.metrics.fbeta_score(labels_actual, labels_pred, beta=0.5) # first way pm.set_stat(ClassificationMetrics.FBETA_SCORE, fbeta_score) pm.set_stat(ClassificationMetrics.FBETA_SCORE, [1, 2, 3]) # second way pm.metrics.fbeta_score(y_true=labels_actual, y_pred=labels_pred, beta=0.5) # should throw error if labels predicted is different length than actuals with pytest.raises(ValueError): labels_prob_missing_values = [0, 0, 0, 1] pm.metrics.fbeta_score(y_true=labels_actual, y_pred=labels_prob_missing_values, beta=0.5) sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0] # testing with sample weights as well pm.metrics.fbeta_score(y_true=labels_actual, y_pred=labels_pred, sample_weight=sample_weight, beta=0.5) labels_pred_multiclass = [1, 0, 2, 1, 1, 2] labels_actual_multiclass = [0, 1, 1, 0, 2, 0] fbeta_score = pm.metrics.fbeta_score(y_true=labels_actual_multiclass, y_pred=labels_pred_multiclass, labels=[0, 1, 2], average=None, beta=0.5) assert len(fbeta_score) == 3 pm.done()
def _report_stats(self, file_path): self._logger.info(" *** generate stats .. params:{}".format( self._params)) self._logger.info(" *** Source file {}".format(file_path)) # Read the file data = pd.read_csv(file_path, sep=' |,', header=None, skiprows=1) data = data.rename(index=str, columns={ 1: "label", 2: "confidence0", 3: "confidence1" }) prediction_distribution = data['label'].value_counts() column_names = np.array( prediction_distribution.index).astype(str).tolist() # Initialize mlops mlops.init() # Report a bar graph bar = BarGraph().name("Prediction Distribution").cols( np.array(prediction_distribution.index).astype(str).tolist()).data( prediction_distribution.values.tolist()) mlops.set_stat(bar) # Generate an alert on low confidence if the argument is set to true if (self._params["alert"]): index = data.values[:, 1].astype(int) confidence = data.values[:, 2:4] confidence_per_prediction = confidence[:, index][:, 0] * 100 low_conf_percent = len(confidence_per_prediction[ confidence_per_prediction < self._params["confidence"]]) / len( confidence_per_prediction) * 100 if low_conf_percent > self._params["samples"]: msg = "Low confidence: {}% of inferences had confidence below {}%".format( low_conf_percent, self._params["confidence"]) print(msg) mlops.health_alert("Low confidence alert", msg) mlops.done() return []
def test_multiline_graph(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) with pytest.raises(MLOpsException): MultiLineGraph().name("mlt").labels([1, 2, 3]) with pytest.raises(MLOpsException): MultiLineGraph().name("mlt").labels(["g1", "g2"]).data(["aa", "bb"]) with pytest.raises(MLOpsException): MultiLineGraph().name("mlt").data(["aa", "bb"]) with pytest.raises(MLOpsException): mlt = MultiLineGraph().name("mlt").labels(["g1"]).data([55, 66]) pm.set_stat(mlt) mlt = MultiLineGraph().name("mlt").labels(["g1", "g2"]).data([55, 66]) pm.set_stat(mlt) pm.done()
def test_mlops_explained_variance_score_apis(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) labels_pred = [1.0, 0.5, 2.5, 4.75, 7.0, 0.75] labels_actual = [1.5, 0.75, 2.75, 4.5, 7.50, 0.25] evs = sklearn.metrics.explained_variance_score(labels_actual, labels_pred) # first way pm.set_stat(RegressionMetrics.EXPLAINED_VARIANCE_SCORE, evs) # array list is allowed as well pm.set_stat(RegressionMetrics.EXPLAINED_VARIANCE_SCORE, [1, 2, 3]) # second way pm.metrics.explained_variance_score(y_true=labels_actual, y_pred=labels_pred) # should throw error if labels predicted is different length than actuals with pytest.raises(ValueError): labels_pred_missing_values = [1.0, 0.5, 7.0, 0.75] pm.metrics.explained_variance_score(y_true=labels_actual, y_pred=labels_pred_missing_values) sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0] # testing with sample weights as well pm.metrics.explained_variance_score(y_true=labels_actual, y_pred=labels_pred, sample_weight=sample_weight) labels_2d_actual = [[1.0, 0.5], [2.5, 4.75], [7.0, 0.75]] labels_2d_pred = [[1.5, 0.75], [2.75, 4.5], [7.50, 0.25]] # testing where result will be multiple float values evs = pm.metrics.explained_variance_score(y_true=labels_2d_actual, y_pred=labels_2d_pred, multioutput="raw_values") assert len(evs) == 2 pm.done()
def main(): ## MLOPS start # Initialize mlops mlops.init() ## MLOPS end parser = argparse.ArgumentParser() add_parameters(parser) args = parser.parse_args() print("Loading model from: {}".format(args.model_dir)) if os.path.isdir(args.model_dir): print("Found model") else: print("No model found. Exiting.") exit(0) # load the model model = SavedModel(args.model_dir, args.sig_name) # get the input input = MnistStreamInput(args.input_dir, args.total_records, args.random) test_data = input._samples mlops.set_data_distribution_stat(test_data) # track confidence conf_tracker = ConfidenceTracker(args.track_conf, args.conf_thresh, args.conf_percent, args.output_low_conf) # perform inferences on the input infer_loop(model, input, args.output_file, args.stats_interval, conf_tracker) del model del input ### MLOPS start mlops.done() ### MLOPS end print("Inference batch complete") exit(0)
def test_attach(): mlapp_id = "144a045d-c927-4afb-a85c-5224bd68f1bb" ion_instance_id = ION1.ION_INSTANCE_ID ion_node_id = ION1.NODE_1_ID token = ION1.TOKEN set_mlops_env(ion_id=ion_instance_id, ion_node_id=ion_node_id, token=token, model_id=ION1.MODEL_ID) rest_helper = MlOpsRestFactory().get_rest_helper(MLOpsMode.AGENT, mlops_server="localhost", mlops_port="3456", token=token) with requests_mock.mock() as m: m.get(rest_helper.url_get_workflow_instance(ion_instance_id), json=test_workflow_instances) m.get(rest_helper.url_get_ees(), json=test_ee_info) m.get(rest_helper.url_get_agents(), json=test_agents_info) m.get(rest_helper.url_get_model_list(), json=test_models_info) m.get(rest_helper.url_get_health_thresholds(ion_instance_id), json=test_health_info) m.get(rest_helper.url_get_model_stats(ION1.MODEL_ID), json=test_model_stats) m.get(rest_helper.url_get_uuid("model"), json={"id": "model_5906255e-0a3d-4fef-8653-8d41911264fb"}) m.post(rest_helper.url_login(), json={"token": token}) pm.attach(mlapp_id=ION1.ION_INSTANCE_ID, mlops_server="localhost", mlops_port=3456, password="******") mlapp_id_ret = pm.get_mlapp_id() assert (mlapp_id_ret == ION1.ION_ID) mlapp_policy_ret = pm.get_mlapp_policy() assert (str(mlapp_policy_ret) == "Policy:\nhealthThreshold: 0.2\ncanaryThreshold: 0.5\n") pm.done()
def test_mlops_classification_report_apis(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) labels_pred = [1, 0, 1, 1, 1, 0] labels_actual = [0, 1, 0, 0, 0, 1] cr = metrics.classification_report(labels_actual, labels_pred) # first way pm.set_stat(ClassificationMetrics.CLASSIFICATION_REPORT, cr) # second way pm.metrics.classification_report(y_true=labels_actual, y_pred=labels_pred) # should throw error if empty string is provided with pytest.raises(MLOpsStatisticsException): pm.set_stat(ClassificationMetrics.CLASSIFICATION_REPORT, "") # should throw error if None string is provided with pytest.raises(MLOpsStatisticsException): pm.set_stat(ClassificationMetrics.CLASSIFICATION_REPORT, None) # should throw error if weird string with pytest.raises(MLOpsStatisticsException): pm.set_stat(ClassificationMetrics.CLASSIFICATION_REPORT, "Hello ParallelM") # should throw error if labels predicted is different length than actuals with pytest.raises(ValueError): labels_pred_missing_values = [0, 0, 0, 1] pm.metrics.classification_report(y_true=labels_actual, y_pred=labels_pred_missing_values) sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0] target_names = ["class Yes", "class No"] # testing with sample weights as well pm.metrics.classification_report(y_true=labels_actual, y_pred=labels_pred, target_names=target_names, sample_weight=sample_weight) pm.done()
def main(): options = parse_args() if options.output_model is None: raise Exception("Model output arg is None") sc = SparkContext(appName="train-test") pm.init(sc) train_node(options) # In addition we run kmeans and generate data distribution histograms # This is spark specific code so it is called here and not in the train_node code. # We will not save the model but just the stats gen_data_dist_stats(sc) sc.stop() pm.done()
def main(): mlops.init() options = parse_args() # Save the model s = "Hello World" f = tempfile.NamedTemporaryFile() pickle.dump(s, f) f.flush() m = mlops.Model(name=options.model_name, model_format=ModelFormat.BINARY, description=options.model_description) m.set_annotations({"aaa": "my annotations"}) m.set_model_path(f.name) mlops.publish_model(m) mlops.set_stat("model post time, minute", dt.now().minute) mlops.set_stat("posted model size", m.metadata.size) mlops.done()
def test_mlops_auc_apis(): pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE) labels_pred = [1, 0, 1, 1, 1, 0] labels_actual = [0, 1, 0, 0, 0, 1] fpr, tpr, thresholds = sklearn.metrics.roc_curve(labels_actual, labels_pred, pos_label=2) auc = sklearn.metrics.auc(fpr, tpr) # first way pm.set_stat(ClassificationMetrics.AUC, auc) # second way pm.metrics.auc(x=fpr, y=tpr) # should throw error if not numeric number is provided with pytest.raises(MLOpsStatisticsException): pm.set_stat(ClassificationMetrics.AUC, [1, 2, 3]) pm.done()