def test_completed(self, xgboost_training_job):
(reference, resource) = xgboost_training_job
assert k8s.get_resource_exists(reference)
training_job_name = resource["spec"].get("trainingJobName", None)
assert training_job_name is not None
training_job_desc = get_sagemaker_training_job(training_job_name)
training_job_arn = training_job_desc["TrainingJobArn"]
assert k8s.get_resource_arn(resource) == training_job_arn
assert training_job_desc[
"TrainingJobStatus"] == cfg.JOB_STATUS_INPROGRESS
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
assert_training_status_in_sync(training_job_name, reference,
cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
# model artifact URL is populated
resource = k8s.get_resource(reference)
resource["status"]["modelArtifacts"]["s3ModelArtifacts"] is not None
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(training_job_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference,
cfg.JOB_DELETE_WAIT_PERIODS,
cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted is True
def test_completed(self, xgboost_hpojob):
(reference, resource) = xgboost_hpojob
assert k8s.get_resource_exists(reference)
hpo_job_name = resource["spec"].get("hyperParameterTuningJobName",
None)
assert hpo_job_name is not None
hpo_sm_desc = get_sagemaker_hpo_job(hpo_job_name)
hpo_arn = hpo_sm_desc["HyperParameterTuningJobArn"]
assert k8s.get_resource_arn(resource) == hpo_arn
assert hpo_sm_desc[
"HyperParameterTuningJobStatus"] == cfg.JOB_STATUS_INPROGRESS
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
self._assert_hpo_status_in_sync(hpo_job_name, reference,
cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(hpo_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference,
cfg.JOB_DELETE_WAIT_PERIODS,
cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted is True
def test_completed(self, kmeans_processing_job):
(reference, resource) = kmeans_processing_job
assert k8s.get_resource_exists(reference)
processing_job_name = resource["spec"].get("processingJobName", None)
assert processing_job_name is not None
processing_job_desc = get_sagemaker_processing_job(processing_job_name)
processing_job_arn = processing_job_desc["ProcessingJobArn"]
assert k8s.get_resource_arn(resource) == processing_job_arn
assert processing_job_desc["ProcessingJobStatus"] == cfg.JOB_STATUS_INPROGRESS
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
self._assert_processing_status_in_sync(
processing_job_name, reference, cfg.JOB_STATUS_COMPLETED
)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(processing_job_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference, cfg.JOB_DELETE_WAIT_PERIODS, cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted is True
def test_create_feature_group(self, feature_group):
"""Tests that a feature group can be created and deleted
using the Feature Group Controller.
"""
(reference, resource) = feature_group
assert k8s.get_resource_exists(reference)
feature_group_name = resource["spec"].get("featureGroupName", None)
assert feature_group_name is not None
feature_group_sm_desc = get_sagemaker_feature_group(feature_group_name)
feature_group_arn = feature_group_sm_desc["FeatureGroupArn"]
assert k8s.get_resource_arn(resource) == feature_group_arn
assert feature_group_sm_desc[
"FeatureGroupStatus"] == FEATURE_GROUP_STATUS_CREATING
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
self._assert_feature_group_status_in_sync(
feature_group_name, reference, FEATURE_GROUP_STATUS_CREATED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(feature_group_arn, resource_tags)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference, WAIT_PERIOD_COUNT,
WAIT_PERIOD_LENGTH)
assert deleted
assert get_sagemaker_feature_group(feature_group_name) is None
def test_smoke(
self, sagemaker_client, xgboost_churn_model_explainability_job_definition
):
(reference, resource) = xgboost_churn_model_explainability_job_definition
assert k8s.get_resource_exists(reference)
job_definition_name = resource["spec"].get("jobDefinitionName")
job_definition_desc = get_sagemaker_model_explainability_job_definition(
sagemaker_client, job_definition_name
)
job_definition_arn = job_definition_desc["JobDefinitionArn"]
assert k8s.get_resource_arn(resource) == job_definition_arn
# random sleep before we check for tags to reduce test flakyness
time.sleep(cfg.TAG_DELAY_SLEEP)
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(job_definition_arn, resource_tags)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference, 3, 10)
assert deleted
assert (
get_sagemaker_model_explainability_job_definition(
sagemaker_client, job_definition_name
)
is None
)
def test_create_model_package_group(self, xgboost_model_package_group):
(reference, resource) = xgboost_model_package_group
assert k8s.get_resource_exists(reference)
model_package_group_name = resource["spec"].get(
"modelPackageGroupName", None)
assert model_package_group_name is not None
model_package_group_sm_desc = get_sagemaker_model_package_group(
model_package_group_name)
model_package_group_arn = model_package_group_sm_desc[
"ModelPackageGroupArn"]
assert k8s.get_resource_arn(resource) == model_package_group_arn
self._assert_model_package_group_status_in_sync(
model_package_group_name, reference, cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(model_package_group_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference,
cfg.DELETE_WAIT_PERIOD,
cfg.DELETE_WAIT_LENGTH)
assert deleted is True
assert get_sagemaker_model_package_group(
model_package_group_name) is None
def test_create_endpoint_config(self, single_variant_config):
(reference, resource) = single_variant_config
assert k8s.get_resource_exists(reference)
config_name = resource["spec"].get("endpointConfigName", None)
endpoint_config_desc = get_sagemaker_endpoint_config(config_name)
endpoint_arn = endpoint_config_desc["EndpointConfigArn"]
assert k8s.get_resource_arn(resource) == endpoint_arn
# random sleep before we check for tags to reduce test flakyness
time.sleep(cfg.TAG_DELAY_SLEEP)
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(endpoint_arn, resource_tags)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference, 3, 10)
assert deleted
assert get_sagemaker_endpoint_config(config_name) is None
def test_completed(self, xgboost_training_job_debugger):
(reference, resource) = xgboost_training_job_debugger
assert k8s.get_resource_exists(reference)
training_job_name = resource["spec"].get("trainingJobName", None)
assert training_job_name is not None
training_job_desc = get_sagemaker_training_job(training_job_name)
training_job_arn = training_job_desc["TrainingJobArn"]
resource_arn = k8s.get_resource_arn(resource)
if resource_arn is None:
logging.error(
f"ARN for this resource is None, resource status is: {resource['status']}"
)
assert resource_arn == training_job_arn
assert training_job_desc[
"TrainingJobStatus"] == cfg.JOB_STATUS_INPROGRESS
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
assert_training_status_in_sync(training_job_name, reference,
cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
# Assert debugger rule evaluation completed
self._assert_training_rule_eval_status_in_sync(
training_job_name, "DebugRule", reference,
cfg.RULE_STATUS_COMPLETED)
# Assert profiler rule evaluation completed
self._assert_training_rule_eval_status_in_sync(
training_job_name, "ProfilerRule", reference,
cfg.RULE_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(training_job_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference,
cfg.JOB_DELETE_WAIT_PERIODS,
cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted is True
def test_create_model(self, xgboost_model):
(reference, resource) = xgboost_model
assert k8s.get_resource_exists(reference)
model_name = resource["spec"].get("modelName", None)
model_desc = get_sagemaker_model(model_name)
model_arn = model_desc["ModelArn"]
assert k8s.get_resource_arn(resource) == model_arn
# random sleep before we check for tags to reduce test flakyness
time.sleep(cfg.TAG_DELAY_SLEEP)
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(model_arn, resource_tags)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference, 3, 10)
assert deleted
assert get_sagemaker_model(model_name) is None
def create_endpoint_test(self, xgboost_endpoint):
(reference, resource, _) = xgboost_endpoint
assert k8s.get_resource_exists(reference)
# endpoint has correct arn and status
endpoint_name = resource["spec"].get("endpointName", None)
assert endpoint_name is not None
endpoint_desc = get_sagemaker_endpoint(endpoint_name)
endpoint_arn = endpoint_desc["EndpointArn"]
assert k8s.get_resource_arn(resource) == endpoint_arn
# endpoint transitions Creating -> InService state
assert_endpoint_status_in_sync(endpoint_name, reference,
cfg.ENDPOINT_STATUS_CREATING)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
assert_endpoint_status_in_sync(endpoint_name, reference,
cfg.ENDPOINT_STATUS_INSERVICE)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(endpoint_arn, resource_tags)
def test_unversioned_model_package_completed(
self, xgboost_unversioned_model_package):
(reference, resource) = xgboost_unversioned_model_package
assert k8s.get_resource_exists(reference)
model_package_name = resource["spec"].get("modelPackageName", None)
assert model_package_name is not None
model_package_desc = get_sagemaker_model_package(model_package_name)
model_package_arn = model_package_desc["ModelPackageArn"]
if k8s.get_resource_arn(resource) is None:
logging.error(
f"ARN for this resource is None, resource status is: {resource['status']}"
)
assert k8s.get_resource_arn(resource) == model_package_arn
self._assert_model_package_status_in_sync(model_package_name,
reference,
cfg.JOB_STATUS_INPROGRESS)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
self._assert_model_package_status_in_sync(model_package_name,
reference,
cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(model_package_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference, DELETE_WAIT_PERIOD,
DELETE_WAIT_LENGTH)
assert deleted is True
assert get_sagemaker_model_package(model_package_name) is None
def test_smoke(
self, sagemaker_client, xgboost_churn_data_quality_monitoring_schedule
):
(reference, resource, spec) = xgboost_churn_data_quality_monitoring_schedule
assert k8s.get_resource_exists(reference)
monitoring_schedule_name = resource["spec"].get("monitoringScheduleName")
monitoring_schedule_desc = get_sagemaker_monitoring_schedule(
sagemaker_client, monitoring_schedule_name
)
monitoring_schedule_arn = monitoring_schedule_desc["MonitoringScheduleArn"]
assert k8s.get_resource_arn(resource) == monitoring_schedule_arn
# scheule transitions Pending -> Scheduled state
# Pending status is shortlived only for 30 seconds because baselining job has already been run
# remove the checks for Pending status if the test is flaky because of this
# as the main objective is to test for Scheduled status
# OR
# create the schedule with a on-going baseline job where it waits for the baselining job to complete
assert (
wait_resource_monitoring_schedule_status(
reference, self.STATUS_PENDING, 5, 2
)
== self.STATUS_PENDING
)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False", 5, 2)
self._assert_monitoring_schedule_status_in_sync(
sagemaker_client, monitoring_schedule_name, reference, self.STATUS_SCHEDULED
)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(monitoring_schedule_arn, resource_tags)
# Update the resource
new_cron_expression = "cron(0 * * * ? *)"
spec["spec"]["monitoringScheduleConfig"]["scheduleConfig"][
"scheduleExpression"
] = new_cron_expression
resource = k8s.patch_custom_resource(reference, spec)
resource = k8s.wait_resource_consumed_by_controller(reference)
assert resource is not None
self._assert_monitoring_schedule_status_in_sync(
sagemaker_client, monitoring_schedule_name, reference, self.STATUS_SCHEDULED
)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
latest_schedule = get_sagemaker_monitoring_schedule(
sagemaker_client, monitoring_schedule_name
)
assert (
latest_schedule["MonitoringScheduleConfig"]["ScheduleConfig"][
"ScheduleExpression"
]
== new_cron_expression
)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference, cfg.DELETE_WAIT_PERIOD, cfg.DELETE_WAIT_LENGTH)
assert deleted
assert get_sagemaker_monitoring_schedule(sagemaker_client, monitoring_schedule_name) is None