def main():
options = parse_args()
sc = SparkContext(appName="predict-test")
pm.init(sc)
predict_node(options)
sc.stop()
pm.done()
def db_to_df(engine, table):
"""
Save DataFrame to Database
"""
mlops.init()
df_sink = pandas.read_sql("{} {}".format(SELECT_STATEMENT, table), con = engine)
mlops.set_stat(table, df_sink.shape[0])
mlops.done()
return(df_sink, df_sink.shape[0])
def _materialize(self, parent_data_objs, user_data):
# Initialize MLOps Library
mlops.init()
df_data = parent_data_objs[0]
df_clean = do_nan_removal(df_data)
get_data_distribution_stat(df_clean)
# Terminate MLOPs
mlops.done()
return [df_clean]
def test_set_stat_basic():
with pytest.raises(MLOpsException):
pm.set_stat(name=None, data=None)
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
pm.set_stat("st1", data=5.5, category=StatCategory.TIME_SERIES)
pm.set_stat("st1", data=5.5)
pm.done()
def test_mlops_structure_api():
ion_instance_id = ION1.ION_INSTANCE_ID
ion_node_id = ION1.NODE_1_ID
token = ION1.TOKEN
set_mlops_env(ion_id=ion_instance_id,
ion_node_id=ion_node_id,
token=token,
model_id=ION1.MODEL_ID)
rest_helper = MlOpsRestFactory().get_rest_helper(MLOpsMode.AGENT,
mlops_server="localhost",
mlops_port="3456",
token=token)
rest_helper.set_prefix(Constants.URL_MLOPS_PREFIX)
with requests_mock.mock() as m:
m.get(rest_helper.url_get_workflow_instance(ion_instance_id),
json=test_workflow_instances)
m.get(rest_helper.url_get_ees(), json=test_ee_info)
m.get(rest_helper.url_get_agents(), json=test_agents_info)
m.get(rest_helper.url_get_model_list(), json=test_models_info)
m.get(rest_helper.url_get_health_thresholds(ion_instance_id),
json=test_health_info)
m.get(rest_helper.url_get_model_stats(ION1.MODEL_ID),
json=test_model_stats)
m.get(rest_helper.url_get_uuid("model"),
json={"id": "model_5906255e-0a3d-4fef-8653-8d41911264fb"})
pm.init(ctx=None, mlops_mode=MLOpsMode.AGENT)
assert pm.get_mlapp_id() == ION1.ION_ID
assert pm.get_mlapp_name() == ION1.ION_NAME
curr_node = pm.get_current_node()
assert curr_node.id == ion_node_id
nodes = pm.get_nodes()
assert len(nodes) == 2
node0 = pm.get_node('1')
assert node0 is not None
assert node0.pipeline_pattern_id == ION1.PIPELINE_PATTERN_ID_1
assert node0.pipeline_instance_id == ION1.PIPELINE_INST_ID_1
node0_agents = pm.get_agents('1')
assert len(node0_agents) == 1
assert node0_agents[0].id == ION1.AGENT_ID_0
assert node0_agents[0].hostname == 'localhost'
agent = pm.get_agent('1', ION1.AGENT_ID_0)
assert agent.id == ION1.AGENT_ID_0
assert agent.hostname == 'localhost'
model = pm.current_model()
assert model is not None
assert model.metadata.modelId == ION1.MODEL_ID
pm.done()
def _cleanup_on_exist(self):
if self._ml_engine:
self._ml_engine.stop()
if mlops_loaded and mlops.init_called:
mlops.done()
if self._ml_engine:
self._ml_engine.cleanup()
def test_set_events_api():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
pm._set_api_test_mode()
with pytest.raises(MLOpsException):
pm.set_event(name=None, data=None, type=EventType.System)
with pytest.raises(MLOpsException):
pm.set_event(name="ssss", data=None, type=None)
pm.done()
def test_data_distribution_stat_api(generate_da_with_missing_data):
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
pm._set_api_test_mode()
# basic test
data = np.array([[1, 2], [3, 4]])
pm.set_data_distribution_stat(data)
# test with column missing
blah = pd.read_csv(generate_da_with_missing_data)
pm.set_data_distribution_stat(blah)
pm.done()
def test_data_distribution_stat_api_spark(spark_session, generate_da_with_missing_data):
sc = spark_session.sparkContext
pm.init(ctx=sc, mlops_mode=MLOpsMode.STAND_ALONE)
pm._set_api_test_mode()
pdf = pd.read_csv(generate_da_with_missing_data)
spark_df = spark_session.createDataFrame(pdf)
pm.set_data_distribution_stat(data=spark_df)
sc.stop()
pm.done()
def test_init_done():
"""
Testing api for information such as ION id, ion name and such
:return:
"""
with pytest.raises(MLOpsException):
pm.set_stat("st1", 5.5)
with pytest.raises(MLOpsException):
pm.done()
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
pm.done()
def read_file_to_df(self, filepath):
"""
Read file and return DataFrame
"""
mlops.init()
if not os.path.exists(filepath):
self._logger.info("stderr- failed to find {}".format(filepath), file=sys.stderr)
raise Exception("file path does not exist: {}".format(filepath))
test_data = pandas.read_csv(filepath)
mlops.done()
return test_data
def test_mlops_roc_curve_apis():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
labels_pred_prob = [0.9, 0.4, 0.6, 0.9, 0.1, 0.9]
labels_actual = [0, 1, 0, 0, 0, 1]
fpr, tpr, thresholds = sklearn.metrics.roc_curve(labels_actual,
labels_pred_prob,
pos_label=1,
sample_weight=None,
drop_intermediate=True)
roc_auc_score = sklearn.metrics.roc_auc_score(labels_actual,
labels_pred_prob)
graph_label_str = "AUC: {}".format(roc_auc_score)
# first way
pm.set_stat(ClassificationMetrics.PRECISION_RECALL_CURVE, [tpr, fpr])
pm.set_stat(ClassificationMetrics.PRECISION_RECALL_CURVE, [tpr, fpr],
legend=graph_label_str)
# second way
pm.metrics.roc_curve(labels_actual,
labels_pred_prob,
pos_label=1,
sample_weight=None,
drop_intermediate=True)
# should throw error if labels predicted is different length than actuals
with pytest.raises(ValueError):
labels_prob_missing_values = [0.0, 0.9, 1.0, 0.85]
pm.metrics.roc_curve(labels_actual,
labels_prob_missing_values,
pos_label=1,
sample_weight=None,
drop_intermediate=True)
sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0]
# testing with sample weights as well
pm.metrics.roc_curve(labels_actual,
labels_pred_prob,
pos_label=1,
sample_weight=sample_weight,
drop_intermediate=True)
pm.done()
def main():
options = parse_args()
sc = SparkContext(appName="health-test")
pm.init(sc)
print("MLOps testing")
print("My ION Node id: {}".format(mlops.get_current_node().id))
print("My ION Node name: {}".format(mlops.get_current_node().name))
run_mlops_tests(package_to_scan=parallelm.mlops.e2e_tests.health_node,
test_to_run=options.test_name)
sc.stop()
pm.done()
def main():
sc = pyspark.SparkContext(appName="PySparkWordCount")
mlops.init(sc)
options = parse_args()
if options.words_file is None:
print("Bad option - no file was provided")
mlops.set_stat("got_file", 0)
else:
mlops.set_stat("got_file", 1)
count_words(sc, options.words_file)
sc.stop()
mlops.done()
def main():
sc = SparkContext(appName="fake ab")
pm.init(sc)
print("Test")
try:
pm.set_stat("samples", 10000, st.TIME_SERIES)
pm.set_stat("conversions", randint(1, 10000), st.TIME_SERIES)
except Exception as e:
print("Got exception while getting stats: {}".format(e))
pm.set_stat("error", 1, st.TIME_SERIES)
sc.stop()
pm.done()
def test_mlops_confusion_metrics_apis():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
labels_pred = [1, 0, 1, 1, 1, 0]
labels_actual = [0, 1, 0, 0, 0, 1]
labels_ordered = [0, 1]
cm = metrics.confusion_matrix(labels_actual,
labels_pred,
labels=labels_ordered)
# first way
pm.set_stat(ClassificationMetrics.CONFUSION_MATRIX,
cm,
labels=labels_ordered)
# second way
pm.metrics.confusion_matrix(y_true=labels_actual,
y_pred=labels_pred,
labels=labels_ordered)
# should throw error if labels are not provided
with pytest.raises(MLOpsStatisticsException):
pm.set_stat(ClassificationMetrics.CONFUSION_MATRIX, cm)
with pytest.raises(MLOpsStatisticsException):
pm.metrics.confusion_matrix(y_true=labels_actual,
y_pred=labels_pred,
labels=None)
# should throw error if labels predicted is different length than actuals
with pytest.raises(ValueError):
labels_pred_missing_values = [0, 0, 0, 1]
pm.metrics.confusion_matrix(y_true=labels_actual,
y_pred=labels_pred_missing_values,
labels=None)
sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0]
# testing with sample weights as well
pm.metrics.confusion_matrix(y_true=labels_actual,
y_pred=labels_pred,
labels=labels_ordered,
sample_weight=sample_weight)
pm.done()
def test_mlops_precision_recall_curve_apis():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
labels_pred_prob = [0.9, 0.4, 0.6, 0.9, 0.1, 0.9]
labels_actual = [0, 1, 0, 0, 0, 1]
precision, recall, thresholds = sklearn.metrics.precision_recall_curve(
y_true=labels_actual, probas_pred=labels_pred_prob)
# first way
pm.set_stat(ClassificationMetrics.PRECISION_RECALL_CURVE,
[precision, recall])
pm.set_stat(ClassificationMetrics.PRECISION_RECALL_CURVE,
[precision, recall],
legend="Precision Recall")
# second way
pm.metrics.precision_recall_curve(y_true=labels_actual,
probas_pred=labels_pred_prob)
# should throw error if labels predicted is different length than actuals
with pytest.raises(ValueError):
labels_prob_missing_values = [0.0, 0.9, 1.0, 0.85]
pm.metrics.precision_recall_curve(
y_true=labels_actual, probas_pred=labels_prob_missing_values)
sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0]
# testing with sample weights as well
pm.metrics.precision_recall_curve(y_true=labels_actual,
probas_pred=labels_pred_prob,
sample_weight=sample_weight)
# testing with pos label as well
pm.metrics.precision_recall_curve(y_true=labels_actual,
probas_pred=labels_pred_prob,
sample_weight=sample_weight,
pos_label=1)
# testing with average as well
pm.metrics.precision_recall_curve(y_true=labels_actual,
probas_pred=labels_pred_prob,
sample_weight=sample_weight,
pos_label=1,
average="micro")
pm.done()
def test_mlops_mean_squared_log_error_apis():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
labels_pred = [1.0, 0.5, 2.5, 4.75, 7.0, 0.75]
labels_actual = [1.5, 0.75, 2.75, 4.5, 7.50, 0.25]
msle = sklearn.metrics.mean_squared_log_error(labels_actual, labels_pred)
# first way
pm.set_stat(RegressionMetrics.MEAN_SQUARED_LOG_ERROR, msle)
pm.set_stat(RegressionMetrics.MEAN_SQUARED_LOG_ERROR, [1, 2, 3])
# second way
pm.metrics.mean_squared_log_error(y_true=labels_actual, y_pred=labels_pred)
# should throw error if labels predicted is different length than actuals
with pytest.raises(ValueError):
labels_pred_missing_values = [1.0, 0.5, 7.0, 0.75]
pm.metrics.mean_squared_log_error(y_true=labels_actual,
y_pred=labels_pred_missing_values)
# should throw error if labels contain negative values
with pytest.raises(ValueError):
labels_pred_neg = [1.0, -0.5, 2.5, 4.75, 7.0, 0.75]
labels_actual_neg = [1.5, -0.75, 2.75, 4.5, 7.50, 0.25]
pm.metrics.mean_squared_log_error(y_true=labels_actual_neg,
y_pred=labels_pred_neg)
sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0]
# testing with sample weights as well
pm.metrics.mean_squared_log_error(y_true=labels_actual,
y_pred=labels_pred,
sample_weight=sample_weight)
labels_2d_actual = [[1.0, 0.5], [2.5, 4.75], [7.0, 0.75]]
labels_2d_pred = [[1.5, 0.75], [2.75, 4.5], [7.50, 0.25]]
msle = pm.metrics.mean_squared_log_error(y_true=labels_2d_actual,
y_pred=labels_2d_pred,
multioutput="raw_values")
assert len(msle) == 2
pm.done()
def main():
mlops.init()
options = parse_args()
for i in range(10):
model = mlops.get_last_approved_model()
if model:
mlops.set_stat("model fetched", 10)
mlops.set_stat("fetched model size", model.metadata.size)
print("FETCHED MODEL: {}".format(model.metadata))
break
else:
mlops.set_stat("model fetched", 0)
time.sleep(1)
mlops.done()
def test_mlops_precision_score_apis():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
labels_pred = [1, 0, 1, 1, 1, 0]
labels_actual = [0, 1, 0, 0, 0, 1]
precision_score = sklearn.metrics.precision_score(labels_actual,
labels_pred)
# first way
pm.set_stat(ClassificationMetrics.PRECISION_SCORE, precision_score)
# second way
pm.metrics.precision_score(y_true=labels_actual, y_pred=labels_pred)
sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0]
# testing with sample weights as well
pm.metrics.precision_score(y_true=labels_actual,
y_pred=labels_pred,
sample_weight=sample_weight)
labels_pred_multiclass = [1, 0, 2, 1, 1, 2]
labels_actual_multiclass = [0, 1, 1, 0, 2, 0]
# first way where precision score is array of values per class.
precision_score = sklearn.metrics.precision_score(
y_true=labels_actual_multiclass,
y_pred=labels_pred_multiclass,
pos_label=2,
labels=[0, 1, 2],
average=None)
pm.set_stat(ClassificationMetrics.PRECISION_SCORE, precision_score)
# second way
precision_score = pm.metrics.precision_score(
y_true=labels_actual_multiclass,
y_pred=labels_pred_multiclass,
pos_label=2,
labels=[0, 1, 2],
average=None)
assert len(precision_score) == 3
pm.done()
def test_mlops_fbeta_score_apis():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
labels_pred = [1, 0, 1, 1, 1, 0]
labels_actual = [0, 1, 0, 0, 0, 1]
fbeta_score = sklearn.metrics.fbeta_score(labels_actual,
labels_pred,
beta=0.5)
# first way
pm.set_stat(ClassificationMetrics.FBETA_SCORE, fbeta_score)
pm.set_stat(ClassificationMetrics.FBETA_SCORE, [1, 2, 3])
# second way
pm.metrics.fbeta_score(y_true=labels_actual, y_pred=labels_pred, beta=0.5)
# should throw error if labels predicted is different length than actuals
with pytest.raises(ValueError):
labels_prob_missing_values = [0, 0, 0, 1]
pm.metrics.fbeta_score(y_true=labels_actual,
y_pred=labels_prob_missing_values,
beta=0.5)
sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0]
# testing with sample weights as well
pm.metrics.fbeta_score(y_true=labels_actual,
y_pred=labels_pred,
sample_weight=sample_weight,
beta=0.5)
labels_pred_multiclass = [1, 0, 2, 1, 1, 2]
labels_actual_multiclass = [0, 1, 1, 0, 2, 0]
fbeta_score = pm.metrics.fbeta_score(y_true=labels_actual_multiclass,
y_pred=labels_pred_multiclass,
labels=[0, 1, 2],
average=None,
beta=0.5)
assert len(fbeta_score) == 3
pm.done()
def _report_stats(self, file_path):
self._logger.info(" *** generate stats .. params:{}".format(
self._params))
self._logger.info(" *** Source file {}".format(file_path))
# Read the file
data = pd.read_csv(file_path, sep=' |,', header=None, skiprows=1)
data = data.rename(index=str,
columns={
1: "label",
2: "confidence0",
3: "confidence1"
})
prediction_distribution = data['label'].value_counts()
column_names = np.array(
prediction_distribution.index).astype(str).tolist()
# Initialize mlops
mlops.init()
# Report a bar graph
bar = BarGraph().name("Prediction Distribution").cols(
np.array(prediction_distribution.index).astype(str).tolist()).data(
prediction_distribution.values.tolist())
mlops.set_stat(bar)
# Generate an alert on low confidence if the argument is set to true
if (self._params["alert"]):
index = data.values[:, 1].astype(int)
confidence = data.values[:, 2:4]
confidence_per_prediction = confidence[:, index][:, 0] * 100
low_conf_percent = len(confidence_per_prediction[
confidence_per_prediction < self._params["confidence"]]) / len(
confidence_per_prediction) * 100
if low_conf_percent > self._params["samples"]:
msg = "Low confidence: {}% of inferences had confidence below {}%".format(
low_conf_percent, self._params["confidence"])
print(msg)
mlops.health_alert("Low confidence alert", msg)
mlops.done()
return []
def test_multiline_graph():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
with pytest.raises(MLOpsException):
MultiLineGraph().name("mlt").labels([1, 2, 3])
with pytest.raises(MLOpsException):
MultiLineGraph().name("mlt").labels(["g1", "g2"]).data(["aa", "bb"])
with pytest.raises(MLOpsException):
MultiLineGraph().name("mlt").data(["aa", "bb"])
with pytest.raises(MLOpsException):
mlt = MultiLineGraph().name("mlt").labels(["g1"]).data([55, 66])
pm.set_stat(mlt)
mlt = MultiLineGraph().name("mlt").labels(["g1", "g2"]).data([55, 66])
pm.set_stat(mlt)
pm.done()
def test_mlops_explained_variance_score_apis():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
labels_pred = [1.0, 0.5, 2.5, 4.75, 7.0, 0.75]
labels_actual = [1.5, 0.75, 2.75, 4.5, 7.50, 0.25]
evs = sklearn.metrics.explained_variance_score(labels_actual, labels_pred)
# first way
pm.set_stat(RegressionMetrics.EXPLAINED_VARIANCE_SCORE, evs)
# array list is allowed as well
pm.set_stat(RegressionMetrics.EXPLAINED_VARIANCE_SCORE, [1, 2, 3])
# second way
pm.metrics.explained_variance_score(y_true=labels_actual,
y_pred=labels_pred)
# should throw error if labels predicted is different length than actuals
with pytest.raises(ValueError):
labels_pred_missing_values = [1.0, 0.5, 7.0, 0.75]
pm.metrics.explained_variance_score(y_true=labels_actual,
y_pred=labels_pred_missing_values)
sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0]
# testing with sample weights as well
pm.metrics.explained_variance_score(y_true=labels_actual,
y_pred=labels_pred,
sample_weight=sample_weight)
labels_2d_actual = [[1.0, 0.5], [2.5, 4.75], [7.0, 0.75]]
labels_2d_pred = [[1.5, 0.75], [2.75, 4.5], [7.50, 0.25]]
# testing where result will be multiple float values
evs = pm.metrics.explained_variance_score(y_true=labels_2d_actual,
y_pred=labels_2d_pred,
multioutput="raw_values")
assert len(evs) == 2
pm.done()
def main():
## MLOPS start
# Initialize mlops
mlops.init()
## MLOPS end
parser = argparse.ArgumentParser()
add_parameters(parser)
args = parser.parse_args()
print("Loading model from: {}".format(args.model_dir))
if os.path.isdir(args.model_dir):
print("Found model")
else:
print("No model found. Exiting.")
exit(0)
# load the model
model = SavedModel(args.model_dir, args.sig_name)
# get the input
input = MnistStreamInput(args.input_dir, args.total_records, args.random)
test_data = input._samples
mlops.set_data_distribution_stat(test_data)
# track confidence
conf_tracker = ConfidenceTracker(args.track_conf, args.conf_thresh,
args.conf_percent, args.output_low_conf)
# perform inferences on the input
infer_loop(model, input, args.output_file, args.stats_interval,
conf_tracker)
del model
del input
### MLOPS start
mlops.done()
### MLOPS end
print("Inference batch complete")
exit(0)
def test_attach():
mlapp_id = "144a045d-c927-4afb-a85c-5224bd68f1bb"
ion_instance_id = ION1.ION_INSTANCE_ID
ion_node_id = ION1.NODE_1_ID
token = ION1.TOKEN
set_mlops_env(ion_id=ion_instance_id,
ion_node_id=ion_node_id,
token=token,
model_id=ION1.MODEL_ID)
rest_helper = MlOpsRestFactory().get_rest_helper(MLOpsMode.AGENT,
mlops_server="localhost",
mlops_port="3456",
token=token)
with requests_mock.mock() as m:
m.get(rest_helper.url_get_workflow_instance(ion_instance_id),
json=test_workflow_instances)
m.get(rest_helper.url_get_ees(), json=test_ee_info)
m.get(rest_helper.url_get_agents(), json=test_agents_info)
m.get(rest_helper.url_get_model_list(), json=test_models_info)
m.get(rest_helper.url_get_health_thresholds(ion_instance_id),
json=test_health_info)
m.get(rest_helper.url_get_model_stats(ION1.MODEL_ID),
json=test_model_stats)
m.get(rest_helper.url_get_uuid("model"),
json={"id": "model_5906255e-0a3d-4fef-8653-8d41911264fb"})
m.post(rest_helper.url_login(), json={"token": token})
pm.attach(mlapp_id=ION1.ION_INSTANCE_ID,
mlops_server="localhost",
mlops_port=3456,
password="******")
mlapp_id_ret = pm.get_mlapp_id()
assert (mlapp_id_ret == ION1.ION_ID)
mlapp_policy_ret = pm.get_mlapp_policy()
assert (str(mlapp_policy_ret) ==
"Policy:\nhealthThreshold: 0.2\ncanaryThreshold: 0.5\n")
pm.done()
def test_mlops_classification_report_apis():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
labels_pred = [1, 0, 1, 1, 1, 0]
labels_actual = [0, 1, 0, 0, 0, 1]
cr = metrics.classification_report(labels_actual, labels_pred)
# first way
pm.set_stat(ClassificationMetrics.CLASSIFICATION_REPORT, cr)
# second way
pm.metrics.classification_report(y_true=labels_actual, y_pred=labels_pred)
# should throw error if empty string is provided
with pytest.raises(MLOpsStatisticsException):
pm.set_stat(ClassificationMetrics.CLASSIFICATION_REPORT, "")
# should throw error if None string is provided
with pytest.raises(MLOpsStatisticsException):
pm.set_stat(ClassificationMetrics.CLASSIFICATION_REPORT, None)
# should throw error if weird string
with pytest.raises(MLOpsStatisticsException):
pm.set_stat(ClassificationMetrics.CLASSIFICATION_REPORT,
"Hello ParallelM")
# should throw error if labels predicted is different length than actuals
with pytest.raises(ValueError):
labels_pred_missing_values = [0, 0, 0, 1]
pm.metrics.classification_report(y_true=labels_actual,
y_pred=labels_pred_missing_values)
sample_weight = [0.9, 0.1, 0.5, 0.9, 1.0, 0]
target_names = ["class Yes", "class No"]
# testing with sample weights as well
pm.metrics.classification_report(y_true=labels_actual,
y_pred=labels_pred,
target_names=target_names,
sample_weight=sample_weight)
pm.done()
def main():
options = parse_args()
if options.output_model is None:
raise Exception("Model output arg is None")
sc = SparkContext(appName="train-test")
pm.init(sc)
train_node(options)
# In addition we run kmeans and generate data distribution histograms
# This is spark specific code so it is called here and not in the train_node code.
# We will not save the model but just the stats
gen_data_dist_stats(sc)
sc.stop()
pm.done()
def main():
mlops.init()
options = parse_args()
# Save the model
s = "Hello World"
f = tempfile.NamedTemporaryFile()
pickle.dump(s, f)
f.flush()
m = mlops.Model(name=options.model_name,
model_format=ModelFormat.BINARY,
description=options.model_description)
m.set_annotations({"aaa": "my annotations"})
m.set_model_path(f.name)
mlops.publish_model(m)
mlops.set_stat("model post time, minute", dt.now().minute)
mlops.set_stat("posted model size", m.metadata.size)
mlops.done()
def test_mlops_auc_apis():
pm.init(ctx=None, mlops_mode=MLOpsMode.STAND_ALONE)
labels_pred = [1, 0, 1, 1, 1, 0]
labels_actual = [0, 1, 0, 0, 0, 1]
fpr, tpr, thresholds = sklearn.metrics.roc_curve(labels_actual, labels_pred, pos_label=2)
auc = sklearn.metrics.auc(fpr, tpr)
# first way
pm.set_stat(ClassificationMetrics.AUC, auc)
# second way
pm.metrics.auc(x=fpr, y=tpr)
# should throw error if not numeric number is provided
with pytest.raises(MLOpsStatisticsException):
pm.set_stat(ClassificationMetrics.AUC, [1, 2, 3])
pm.done()
