def model_quality_monitor(sagemaker_session):
monitor = ModelQualityMonitor(
role=ROLE,
instance_count=INSTANCE_COUNT,
instance_type=INSTANCE_TYPE,
volume_size_in_gb=VOLUME_SIZE_IN_GB,
max_runtime_in_seconds=MAX_RUNTIME_IN_SECONDS,
sagemaker_session=sagemaker_session,
env=TEST_ENV,
tags=TEST_TAGS,
)
return monitor
def test_model_quality_monitor(sagemaker_session,
scheduled_model_quality_monitor, endpoint_name,
ground_truth_input):
monitor = scheduled_model_quality_monitor
monitor._wait_for_schedule_changes_to_apply()
# stop it as soon as possible to avoid any execution
monitor.stop_monitoring_schedule()
_verify_monitoring_schedule(
monitor=monitor,
schedule_status="Stopped",
)
_verify_model_quality_job_description(
sagemaker_session=sagemaker_session,
monitor=monitor,
endpoint_name=endpoint_name,
ground_truth_input=ground_truth_input,
)
# attach to schedule
monitoring_schedule_name = monitor.monitoring_schedule_name
job_definition_name = monitor.job_definition_name
monitor = ModelQualityMonitor.attach(
monitor_schedule_name=monitor.monitoring_schedule_name,
sagemaker_session=sagemaker_session,
)
assert monitor.monitoring_schedule_name == monitoring_schedule_name
assert monitor.job_definition_name == job_definition_name
# update schedule
monitor.update_monitoring_schedule(
max_runtime_in_seconds=UPDATED_MAX_RUNTIME_IN_SECONDS,
schedule_cron_expression=UPDATED_CRON)
assert monitor.monitoring_schedule_name == monitoring_schedule_name
assert monitor.job_definition_name != job_definition_name
_verify_monitoring_schedule(monitor=monitor,
schedule_status="Scheduled",
schedule_cron_expression=UPDATED_CRON)
_verify_model_quality_job_description(
sagemaker_session=sagemaker_session,
monitor=monitor,
endpoint_name=endpoint_name,
ground_truth_input=ground_truth_input,
max_runtime_in_seconds=UPDATED_MAX_RUNTIME_IN_SECONDS,
)
# delete schedule
monitor.delete_monitoring_schedule()
def test_run_model_quality_monitor_baseline(
sagemaker_session,
endpoint_name,
data_path,
ground_truth_input,
upload_actual_data,
):
monitor = ModelQualityMonitor(
role=ROLE,
instance_count=INSTANCE_COUNT,
instance_type=INSTANCE_TYPE,
volume_size_in_gb=VOLUME_SIZE_IN_GB,
max_runtime_in_seconds=MAX_RUNTIME_IN_SECONDS,
sagemaker_session=sagemaker_session,
env=TEST_ENV,
tags=TEST_TAGS,
)
baselining_job_name = utils.unique_name_from_base(
"model-quality-baselining-job")
print("Creating baselining job: {}".format(baselining_job_name))
monitor.suggest_baseline(
baseline_dataset=data_path,
dataset_format=DatasetFormat.csv(),
problem_type=PROBLEM_TYPE,
job_name=baselining_job_name,
ground_truth_attribute=HEADER_OF_LABEL,
inference_attribute=HEADER_OF_PREDICTED_LABEL,
)
monitoring_schedule_name = utils.unique_name_from_base(
"model-quality-suggest-baseline")
s3_uri_monitoring_output = os.path.join(
"s3://",
sagemaker_session.default_bucket(),
endpoint_name,
monitoring_schedule_name,
"monitor_output",
)
monitor.create_monitoring_schedule(
endpoint_input=EndpointInput(
endpoint_name=endpoint_name,
destination=ENDPOINT_INPUT_LOCAL_PATH,
start_time_offset=START_TIME_OFFSET,
end_time_offset=END_TIME_OFFSET,
inference_attribute=INFERENCE_ATTRIBUTE,
),
ground_truth_input=ground_truth_input,
problem_type=PROBLEM_TYPE,
output_s3_uri=s3_uri_monitoring_output,
monitor_schedule_name=monitoring_schedule_name,
schedule_cron_expression=CRON,
)
_verify_execution_status(monitor)
_verify_model_quality_job_description(
sagemaker_session=sagemaker_session,
monitor=monitor,
endpoint_name=endpoint_name,
ground_truth_input=ground_truth_input,
)
monitor.delete_monitoring_schedule()
def _generate_model_monitor(self, mm_type: str) -> Optional[ModelMonitor]:
"""Generates a ModelMonitor object
Generates a ModelMonitor object with required config attributes for
QualityCheckStep and ClarifyCheckStep
Args:
mm_type (str): The subclass type of ModelMonitor object.
A valid mm_type should be one of the following: "DefaultModelMonitor",
"ModelQualityMonitor", "ModelBiasMonitor", "ModelExplainabilityMonitor"
Return:
sagemaker.model_monitor.ModelMonitor or None if the mm_type is not valid
"""
if mm_type == "DefaultModelMonitor":
monitor = DefaultModelMonitor(
role=self.role,
instance_count=self.instance_count,
instance_type=self.instance_type,
volume_size_in_gb=self.volume_size_in_gb,
volume_kms_key=self.volume_kms_key,
output_kms_key=self.output_kms_key,
max_runtime_in_seconds=self.max_runtime_in_seconds,
base_job_name=self.base_job_name,
sagemaker_session=self.sagemaker_session,
env=self.env,
tags=self.tags,
network_config=self.network_config,
)
elif mm_type == "ModelQualityMonitor":
monitor = ModelQualityMonitor(
role=self.role,
instance_count=self.instance_count,
instance_type=self.instance_type,
volume_size_in_gb=self.volume_size_in_gb,
volume_kms_key=self.volume_kms_key,
output_kms_key=self.output_kms_key,
max_runtime_in_seconds=self.max_runtime_in_seconds,
base_job_name=self.base_job_name,
sagemaker_session=self.sagemaker_session,
env=self.env,
tags=self.tags,
network_config=self.network_config,
)
elif mm_type == "ModelBiasMonitor":
monitor = ModelBiasMonitor(
role=self.role,
instance_count=self.instance_count,
instance_type=self.instance_type,
volume_size_in_gb=self.volume_size_in_gb,
volume_kms_key=self.volume_kms_key,
output_kms_key=self.output_kms_key,
max_runtime_in_seconds=self.max_runtime_in_seconds,
base_job_name=self.base_job_name,
sagemaker_session=self.sagemaker_session,
env=self.env,
tags=self.tags,
network_config=self.network_config,
)
elif mm_type == "ModelExplainabilityMonitor":
monitor = ModelExplainabilityMonitor(
role=self.role,
instance_count=self.instance_count,
instance_type=self.instance_type,
volume_size_in_gb=self.volume_size_in_gb,
volume_kms_key=self.volume_kms_key,
output_kms_key=self.output_kms_key,
max_runtime_in_seconds=self.max_runtime_in_seconds,
base_job_name=self.base_job_name,
sagemaker_session=self.sagemaker_session,
env=self.env,
tags=self.tags,
network_config=self.network_config,
)
else:
logging.warning(
'Expected model monitor types: "DefaultModelMonitor", "ModelQualityMonitor", '
'"ModelBiasMonitor", "ModelExplainabilityMonitor"')
return None
return monitor
def main(resources, train_data):
# configurarion
AWS_DEFAULT_REGION = os.getenv('AWS_DEFAULT_REGION', 'eu-west-1')
AWS_PROFILE = os.getenv('AWS_PROFILE', 'default')
AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID', None)
AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY', None)
b3_session, sm_client, sm_runtime, sm_session = get_sm_session(
region=AWS_DEFAULT_REGION,
profile_name=AWS_PROFILE,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
BASE_JOB_PREFIX = os.getenv('BASE_JOB_PREFIX', 'sts')
ROLE_ARN = os.getenv('AWS_ROLE', sagemaker.get_execution_role())
outputs = resources
bucket = sm_session.default_bucket()
prefix = "{}/{}".format(BASE_JOB_PREFIX, resources['endpoint']['name'])
if 'monitor' not in resources:
raise ValueError("Monitoring not enabled")
if 's3_capture_upload_path' not in resources['monitor']:
raise ValueError("Monitoring not enabled")
baseline_prefix = prefix + "/baselining"
baseline_data_prefix = baseline_prefix + "/data"
baseline_results_prefix = baseline_prefix + "/results"
baseline_data_uri = "s3://{}/{}".format(bucket, baseline_data_prefix)
baseline_results_uri = "s3://{}/{}".format(bucket, baseline_results_prefix)
outputs['monitor'].update({
'baseline': {
'data_uri': baseline_data_uri,
'results_uri': baseline_results_uri
}
})
_l.info("Baseline data uri: {}".format(baseline_data_uri))
_l.info("Baseline results uri: {}".format(baseline_results_uri))
ground_truth_upload_path = f"s3://{bucket}/{prefix}/ground_truth_data"
_l.info(f"Ground truth uri: {ground_truth_upload_path}")
outputs['monitor'].update({'ground truth uri': ground_truth_upload_path})
# Create a baselining job with training dataset
_l.info("Executing a baselining job with training dataset")
_l.info(f"baseline_data_uri: {train_data['baseline']['validate']}")
my_monitor = ModelQualityMonitor(
role=ROLE_ARN,
sagemaker_session=sm_session,
max_runtime_in_seconds=1800 # 30 minutes
)
my_monitor.suggest_baseline(
baseline_dataset=train_data['baseline']['validate'] + "/baseline.csv",
dataset_format=DatasetFormat.csv(header=True),
problem_type="Regression",
inference_attribute="prediction",
ground_truth_attribute="label",
output_s3_uri=baseline_results_uri,
wait=True)
baseline_job = my_monitor.latest_baselining_job
_l.info("suggested baseline contrains")
_l.info(
pprint.pformat(baseline_job.suggested_constraints().
body_dict["regression_constraints"]))
_l.info("suggested baseline statistics")
_l.info(
pprint.pformat(baseline_job.baseline_statistics().
body_dict["regression_metrics"]))
monitor_schedule_name = (
f"{BASE_JOB_PREFIX}-mq-sch-{datetime.datetime.utcnow():%Y-%m-%d-%H%M}")
_l.info(f"Monitoring schedule name: {monitor_schedule_name}")
outputs['monitor'].update({'schedule_name': monitor_schedule_name})
endpointInput = EndpointInput(
resources['endpoint']['name'],
"/opt/ml/processing/input_data",
inference_attribute='0' # REVIEW:
)
my_monitor.create_monitoring_schedule(
monitor_schedule_name=monitor_schedule_name,
endpoint_input=endpointInput,
output_s3_uri=baseline_results_uri,
problem_type="Regression",
ground_truth_input=ground_truth_upload_path,
constraints=baseline_job.suggested_constraints(),
# run the scheduler hourly
schedule_cron_expression=CronExpressionGenerator.hourly(),
enable_cloudwatch_metrics=True,
)
mq_schedule_details = my_monitor.describe_schedule()
while mq_schedule_details['MonitoringScheduleStatus'] == 'Pending':
_l.info(f'Waiting for {monitor_schedule_name}')
time.sleep(3)
mq_schedule_details = my_monitor.describe_schedule()
_l.debug(
f"Model Quality Monitor - schedule details: {pprint.pformat(mq_schedule_details)}"
)
_l.info(
f"Model Quality Monitor - schedule status: {mq_schedule_details['MonitoringScheduleStatus']}"
)
# save outputs to a file
with open('deploymodel_out.json', 'w') as f:
json.dump(outputs, f, default=json_default)