def add_delayed_metrics(self, uuids, ground_truths, sold_dates):
"""
Add delayed metrics to CML Model Metrics database provided a list of prediction UUID's
and corresponding list of metrics (ground truth values and sold dates).
Args:
uuids (list)
ground_truths (list)
sold_dates (list)
"""
if len(uuids) != len(ground_truths) != len(sold_dates):
raise ValueError(
"UUIDs, ground_truths, and sold_dates must be of same length and correspond by index."
)
for uuid, gt, ds in zip(uuids, ground_truths, sold_dates):
cdsw.track_delayed_metrics(metrics={
"ground_truth": gt,
"date_sold": ds
},
prediction_uuid=uuid)
logger.info(
f"Sucessfully added ground truth values to {len(uuids)} records")
response_labels_sample.append({
"uuid":
response["response"]["uuid"],
"final_label":
churn_error(record["Churn"], percent_counter / percent_max),
"response_label":
response["response"]["prediction"]["probability"] >= 0.5,
"timestamp_ms":
int(round(time.time() * 1000))
})
# The "ground truth" loop adds the updated actual label value and an accuracy measure
# every 100 calls to the model.
for index, vals in enumerate(response_labels_sample):
print("Update {} records".format(index)) if (index % 50 == 0) else None
cdsw.track_delayed_metrics({"final_label": vals['final_label']},
vals['uuid'])
if (index % 100 == 0):
start_timestamp_ms = vals['timestamp_ms']
final_labels = []
response_labels = []
final_labels.append(vals['final_label'])
response_labels.append(vals['response_label'])
if (index % 100 == 99):
print("Adding accuracy metrc")
end_timestamp_ms = vals['timestamp_ms']
accuracy = classification_report(final_labels,
response_labels,
output_dict=True)["accuracy"]
cdsw.track_aggregate_metrics({"accuracy": accuracy},
start_timestamp_ms,
end_timestamp_ms,
input_data[5] = int(input_data[5])
input_data[6] = int(input_data[6])
input_data[9] = int(input_data[9])
input_data_string = ""
for record in input_data[:-1]:
input_data_string = input_data_string + str(record) + ","
input_data_string = input_data_string[:-1]
response = cdsw.call_model(latest_model["accessKey"],
{"feature": input_data_string})
predicted_result.append(
response["response"]["prediction"]["prediction"])
actual_result.append(input_data[-1:][0])
cdsw.track_delayed_metrics({"actual_result": input_data[-1:][0]},
response["response"]["uuid"])
print(str(i) + " adding " + input_data_string)
except:
print("invalid row")
time.sleep(0.2)
end_time_ms = int(math.floor(time.time() * 1000))
accuracy = classification_report(actual_result,
predicted_result,
output_dict=True)['accuracy']
cdsw.track_aggregate_metrics({"accuracy": accuracy},
start_time_ms,
end_time_ms,
model_deployment_crn=Deployment_CRN)
print("adding accuracy measure of" + str(accuracy))
# We can now use the read_metrics function to read the metrics we just
# generated into the current session, by querying by time window.
data = cdsw.read_metrics(model_deployment_crn=model_deployment_crn,
start_timestamp_ms=start_timestamp_ms,
end_timestamp_ms=end_timestamp_ms)
data = data['metrics']
# Now, ground truth is known and we want to track the true value
# corresponding to each prediction above.
score_y = iris.data[:test_size, 3].reshape(-1, 1) # Observed petal width
# Track the true values alongside the corresponding predictions using
# track_delayed_metrics. At the same time, calculate the mean absolute
# prediction error.
mean_absolute_error = 0
n = len(score_y)
for i in range(n):
ground_truth = score_x[i][0]
cdsw.track_delayed_metrics({"actual_result":ground_truth}, uuids[i])
absolute_error = np.abs(ground_truth - predictions[i])
mean_absolute_error += absolute_error / n
# Use the track_aggregate_metrics function to record the mean absolute
# error within the time window where we made the model calls above.
cdsw.track_aggregate_metrics(
{"mean_absolute_error": mean_absolute_error},
start_timestamp_ms,
end_timestamp_ms,
model_deployment_crn="crn:cdp:ml:us-west-1:8a1e15cd-04c2-48aa-8f35-b4a8c11997d3:workspace:e4bc3658-32bb-4dc3-a22d-828150329c76/857bd78e-ec51-48dc-be2b-5582e4a8dbe1"
)
response_labels_sample.append({
"uuid":
response["response"]["uuid"],
"final_label":
churn_error(record["Churn"], percent_counter / percent_max),
"response_label":
response["response"]["prediction"]["probability"] >= 0.5,
"timestamp_ms":
int(round(time.time() * 1000)),
})
# The "ground truth" loop adds the updated actual label value and an accuracy measure
# every 100 calls to the model.
for index, vals in enumerate(response_labels_sample):
print("Update {} records".format(index)) if (index % 50 == 0) else None
cdsw.track_delayed_metrics({"final_label": vals["final_label"]},
vals["uuid"])
if index % 100 == 0:
start_timestamp_ms = vals["timestamp_ms"]
final_labels = []
response_labels = []
final_labels.append(vals["final_label"])
response_labels.append(vals["response_label"])
if index % 100 == 99:
print("Adding accuracy metrc")
end_timestamp_ms = vals["timestamp_ms"]
accuracy = classification_report(final_labels,
response_labels,
output_dict=True)["accuracy"]
cdsw.track_aggregate_metrics(
{"accuracy": accuracy},
start_timestamp_ms,
