def add_delayed_metrics(self, uuids, ground_truths, sold_dates): """ Add delayed metrics to CML Model Metrics database provided a list of prediction UUID's and corresponding list of metrics (ground truth values and sold dates). Args: uuids (list) ground_truths (list) sold_dates (list) """ if len(uuids) != len(ground_truths) != len(sold_dates): raise ValueError( "UUIDs, ground_truths, and sold_dates must be of same length and correspond by index." ) for uuid, gt, ds in zip(uuids, ground_truths, sold_dates): cdsw.track_delayed_metrics(metrics={ "ground_truth": gt, "date_sold": ds }, prediction_uuid=uuid) logger.info( f"Sucessfully added ground truth values to {len(uuids)} records")
response_labels_sample.append({ "uuid": response["response"]["uuid"], "final_label": churn_error(record["Churn"], percent_counter / percent_max), "response_label": response["response"]["prediction"]["probability"] >= 0.5, "timestamp_ms": int(round(time.time() * 1000)) }) # The "ground truth" loop adds the updated actual label value and an accuracy measure # every 100 calls to the model. for index, vals in enumerate(response_labels_sample): print("Update {} records".format(index)) if (index % 50 == 0) else None cdsw.track_delayed_metrics({"final_label": vals['final_label']}, vals['uuid']) if (index % 100 == 0): start_timestamp_ms = vals['timestamp_ms'] final_labels = [] response_labels = [] final_labels.append(vals['final_label']) response_labels.append(vals['response_label']) if (index % 100 == 99): print("Adding accuracy metrc") end_timestamp_ms = vals['timestamp_ms'] accuracy = classification_report(final_labels, response_labels, output_dict=True)["accuracy"] cdsw.track_aggregate_metrics({"accuracy": accuracy}, start_timestamp_ms, end_timestamp_ms,
input_data[5] = int(input_data[5]) input_data[6] = int(input_data[6]) input_data[9] = int(input_data[9]) input_data_string = "" for record in input_data[:-1]: input_data_string = input_data_string + str(record) + "," input_data_string = input_data_string[:-1] response = cdsw.call_model(latest_model["accessKey"], {"feature": input_data_string}) predicted_result.append( response["response"]["prediction"]["prediction"]) actual_result.append(input_data[-1:][0]) cdsw.track_delayed_metrics({"actual_result": input_data[-1:][0]}, response["response"]["uuid"]) print(str(i) + " adding " + input_data_string) except: print("invalid row") time.sleep(0.2) end_time_ms = int(math.floor(time.time() * 1000)) accuracy = classification_report(actual_result, predicted_result, output_dict=True)['accuracy'] cdsw.track_aggregate_metrics({"accuracy": accuracy}, start_time_ms, end_time_ms, model_deployment_crn=Deployment_CRN) print("adding accuracy measure of" + str(accuracy))
# We can now use the read_metrics function to read the metrics we just # generated into the current session, by querying by time window. data = cdsw.read_metrics(model_deployment_crn=model_deployment_crn, start_timestamp_ms=start_timestamp_ms, end_timestamp_ms=end_timestamp_ms) data = data['metrics'] # Now, ground truth is known and we want to track the true value # corresponding to each prediction above. score_y = iris.data[:test_size, 3].reshape(-1, 1) # Observed petal width # Track the true values alongside the corresponding predictions using # track_delayed_metrics. At the same time, calculate the mean absolute # prediction error. mean_absolute_error = 0 n = len(score_y) for i in range(n): ground_truth = score_x[i][0] cdsw.track_delayed_metrics({"actual_result":ground_truth}, uuids[i]) absolute_error = np.abs(ground_truth - predictions[i]) mean_absolute_error += absolute_error / n # Use the track_aggregate_metrics function to record the mean absolute # error within the time window where we made the model calls above. cdsw.track_aggregate_metrics( {"mean_absolute_error": mean_absolute_error}, start_timestamp_ms, end_timestamp_ms, model_deployment_crn="crn:cdp:ml:us-west-1:8a1e15cd-04c2-48aa-8f35-b4a8c11997d3:workspace:e4bc3658-32bb-4dc3-a22d-828150329c76/857bd78e-ec51-48dc-be2b-5582e4a8dbe1" )
response_labels_sample.append({ "uuid": response["response"]["uuid"], "final_label": churn_error(record["Churn"], percent_counter / percent_max), "response_label": response["response"]["prediction"]["probability"] >= 0.5, "timestamp_ms": int(round(time.time() * 1000)), }) # The "ground truth" loop adds the updated actual label value and an accuracy measure # every 100 calls to the model. for index, vals in enumerate(response_labels_sample): print("Update {} records".format(index)) if (index % 50 == 0) else None cdsw.track_delayed_metrics({"final_label": vals["final_label"]}, vals["uuid"]) if index % 100 == 0: start_timestamp_ms = vals["timestamp_ms"] final_labels = [] response_labels = [] final_labels.append(vals["final_label"]) response_labels.append(vals["response_label"]) if index % 100 == 99: print("Adding accuracy metrc") end_timestamp_ms = vals["timestamp_ms"] accuracy = classification_report(final_labels, response_labels, output_dict=True)["accuracy"] cdsw.track_aggregate_metrics( {"accuracy": accuracy}, start_timestamp_ms,