def stage_resource(route_resource):
random_suffix = (''.join(
random.choice(string.ascii_lowercase) for _ in range(6)))
stage_resource_name = test_resource_values[
'STAGE_NAME'] + f'-{random_suffix}'
test_resource_values['STAGE_NAME'] = stage_resource_name
stage_ref, stage_data = helper.stage_ref_and_data(
stage_resource_name=stage_resource_name,
replacement_values=test_resource_values)
if k8s.get_resource_exists(stage_ref):
raise Exception(
f"expected {stage_resource_name} to not exist. Did previous test cleanup?"
)
logging.debug(
f"apigatewayv2 stage resource. name: {stage_resource_name}, data: {stage_data}"
)
k8s.create_custom_resource(stage_ref, stage_data)
cr = k8s.wait_resource_consumed_by_controller(stage_ref)
assert cr is not None
assert k8s.get_resource_exists(stage_ref)
yield stage_ref, cr
k8s.delete_custom_resource(stage_ref)
def api_resource():
random_suffix = (''.join(
random.choice(string.ascii_lowercase) for _ in range(6)))
api_resource_name = test_resource_values['API_NAME'] + f'-{random_suffix}'
test_resource_values['API_NAME'] = api_resource_name
api_ref, api_data = helper.api_ref_and_data(
api_resource_name=api_resource_name,
replacement_values=test_resource_values)
if k8s.get_resource_exists(api_ref):
raise Exception(
f"expected {api_resource_name} to not exist. Did previous test cleanup?"
)
logging.debug(
f"http api resource. name: {api_resource_name}, data: {api_data}")
k8s.create_custom_resource(api_ref, api_data)
cr = k8s.wait_resource_consumed_by_controller(api_ref)
assert cr is not None
assert k8s.get_resource_exists(api_ref)
api_id = cr['status']['apiID']
test_resource_values['API_ID'] = api_id
yield api_ref, cr
k8s.delete_custom_resource(api_ref)
def integration_resource(api_resource):
random_suffix = (''.join(
random.choice(string.ascii_lowercase) for _ in range(6)))
integration_resource_name = test_resource_values[
'INTEGRATION_NAME'] + f'-{random_suffix}'
test_resource_values['INTEGRATION_NAME'] = integration_resource_name
integration_ref, integration_data = helper.integration_ref_and_data(
integration_resource_name=integration_resource_name,
replacement_values=test_resource_values)
if k8s.get_resource_exists(integration_ref):
raise Exception(
f"expected {integration_resource_name} to not exist. Did previous test cleanup?"
)
logging.debug(
f"apigatewayv2 integration resource. name: {integration_resource_name}, data: {integration_data}"
)
k8s.create_custom_resource(integration_ref, integration_data)
cr = k8s.wait_resource_consumed_by_controller(integration_ref)
assert cr is not None
assert k8s.get_resource_exists(integration_ref)
integration_id = cr['status']['integrationID']
test_resource_values['INTEGRATION_ID'] = integration_id
yield integration_ref, cr
k8s.delete_custom_resource(integration_ref)
def route_resource(integration_resource, authorizer_resource):
random_suffix = (''.join(
random.choice(string.ascii_lowercase) for _ in range(6)))
route_resource_name = test_resource_values[
'ROUTE_NAME'] + f'-{random_suffix}'
test_resource_values['ROUTE_NAME'] = route_resource_name
route_ref, route_data = helper.route_ref_and_data(
route_resource_name=route_resource_name,
replacement_values=test_resource_values)
if k8s.get_resource_exists(route_ref):
raise Exception(
f"expected {route_resource_name} to not exist. Did previous test cleanup?"
)
logging.debug(
f"apigatewayv2 route resource. name: {route_resource_name}, data: {route_data}"
)
k8s.create_custom_resource(route_ref, route_data)
cr = k8s.wait_resource_consumed_by_controller(route_ref)
assert cr is not None
assert k8s.get_resource_exists(route_ref)
route_id = cr['status']['routeID']
test_resource_values['ROUTE_ID'] = route_id
yield route_ref, cr
k8s.delete_custom_resource(route_ref)
def test_studio(self, app_fixture):
(
domain_reference,
domain_resource,
domain_spec,
user_profile_reference,
user_profile_resource,
user_profile_spec,
app_reference,
app_resource,
app_spec,
) = app_fixture
assert k8s.get_resource_exists(domain_reference)
assert k8s.get_resource_exists(user_profile_reference)
assert k8s.get_resource_exists(app_reference)
domain_id = domain_resource["status"].get("domainID", None)
user_profile_name = user_profile_resource["spec"]["userProfileName"]
app_type = app_resource["spec"]["appType"]
app_name = app_resource["spec"]["appName"]
assert_app_status_in_sync(
domain_id,
user_profile_name,
app_type,
app_name,
app_reference,
"InService",
)
def test_crud_stage(self, api_resource):
api_ref, api_cr = api_resource
api_id = api_cr['status']['apiID']
test_data = REPLACEMENT_VALUES.copy()
random_suffix = (''.join(
random.choice(string.ascii_lowercase) for _ in range(6)))
stage_name = "ack-test-stage-" + random_suffix
test_data['STAGE_NAME'] = stage_name
test_data['API_ID'] = api_id
stage_ref, stage_data = helper.stage_ref_and_data(
stage_resource_name=stage_name, replacement_values=test_data)
logging.debug(
f"http api stage resource. name: {stage_name}, data: {stage_data}")
# test create
k8s.create_custom_resource(stage_ref, stage_data)
cr = k8s.wait_resource_consumed_by_controller(stage_ref)
assert cr is not None
assert k8s.get_resource_exists(stage_ref)
# Let's check that the HTTP Api integration appears in Amazon API Gateway
apigw_validator.assert_stage_is_present(api_id=api_id,
stage_name=stage_name)
stage_description = test_data['STAGE_DESCRIPTION']
apigw_validator.assert_stage_description(
api_id=api_id,
stage_name=stage_name,
expected_description=stage_description)
# test update
updated_description = 'updated' + stage_description
test_data['STAGE_DESCRIPTION'] = updated_description
updated_stage_resource_data = load_apigatewayv2_resource(
"stage",
additional_replacements=test_data,
)
logging.debug(
f"updated http api stage resource: {updated_stage_resource_data}")
# Update the k8s resource
k8s.patch_custom_resource(stage_ref, updated_stage_resource_data)
time.sleep(UPDATE_WAIT_AFTER_SECONDS)
# Let's check that the HTTP Api stage appears in Amazon API Gateway with updated description
apigw_validator.assert_stage_description(
api_id=api_id,
stage_name=stage_name,
expected_description=updated_description)
# test delete
k8s.delete_custom_resource(stage_ref)
time.sleep(DELETE_WAIT_AFTER_SECONDS)
assert not k8s.get_resource_exists(stage_ref)
# HTTP Api stage should no longer appear in Amazon API Gateway
apigw_validator.assert_stage_is_deleted(api_id=api_id,
stage_name=stage_name)
def test_crud_httpapi_using_import(self):
test_data = REPLACEMENT_VALUES.copy()
random_suffix = (''.join(
random.choice(string.ascii_lowercase) for _ in range(6)))
api_name = "ack-test-importapi-" + random_suffix
test_data['API_NAME'] = api_name
test_data['API_TITLE'] = api_name
api_ref, api_data = helper.import_api_ref_and_data(
api_resource_name=api_name, replacement_values=test_data)
logging.debug(
f"imported http api resource. name: {api_name}, data: {api_data}")
# test create
k8s.create_custom_resource(api_ref, api_data)
cr = k8s.wait_resource_consumed_by_controller(api_ref)
assert cr is not None
assert k8s.get_resource_exists(api_ref)
api_id = cr['status']['apiID']
# Let's check that the imported HTTP Api appears in Amazon API Gateway
apigw_validator.assert_api_is_present(api_id=api_id)
apigw_validator.assert_api_name(api_id=api_id,
expected_api_name=api_name)
# test update
updated_api_title = 'updated-' + api_name
test_data['API_TITLE'] = updated_api_title
updated_api_resource_data = load_apigatewayv2_resource(
"import_api",
additional_replacements=test_data,
)
logging.debug(
f"updated import http api resource: {updated_api_resource_data}")
# Update the k8s resource
k8s.patch_custom_resource(api_ref, updated_api_resource_data)
time.sleep(UPDATE_WAIT_AFTER_SECONDS)
# Let's check that the HTTP Api appears in Amazon API Gateway with updated title
apigw_validator.assert_api_name(api_id=api_id,
expected_api_name=updated_api_title)
# test delete
k8s.delete_custom_resource(api_ref)
time.sleep(DELETE_WAIT_AFTER_SECONDS)
assert not k8s.get_resource_exists(api_ref)
# HTTP Api should no longer appear in Amazon API Gateway
apigw_validator.assert_api_is_deleted(api_id=api_id)
def xgboost_endpoint(name_suffix, single_variant_config):
endpoint_resource_name = name_suffix
(_, config_resource) = single_variant_config
config_resource_name = config_resource["spec"].get("endpointConfigName",
None)
replacements = REPLACEMENT_VALUES.copy()
replacements["ENDPOINT_NAME"] = endpoint_resource_name
replacements["ENDPOINT_CONFIG_NAME"] = config_resource_name
reference, spec, resource = create_sagemaker_resource(
resource_plural=cfg.ENDPOINT_RESOURCE_PLURAL,
resource_name=endpoint_resource_name,
spec_file="endpoint_base",
replacements=replacements,
)
assert resource is not None
yield (reference, resource, spec)
# Delete the k8s resource if not already deleted by tests
if k8s.get_resource_exists(reference):
# longer wait incase endpoint is in creating/updating status
_, deleted = k8s.delete_custom_resource(reference, 40,
cfg.DELETE_WAIT_LENGTH)
assert deleted
def test_create_model_package_group(self, xgboost_model_package_group):
(reference, resource) = xgboost_model_package_group
assert k8s.get_resource_exists(reference)
model_package_group_name = resource["spec"].get(
"modelPackageGroupName", None)
assert model_package_group_name is not None
model_package_group_sm_desc = get_sagemaker_model_package_group(
model_package_group_name)
model_package_group_arn = model_package_group_sm_desc[
"ModelPackageGroupArn"]
assert k8s.get_resource_arn(resource) == model_package_group_arn
self._assert_model_package_group_status_in_sync(
model_package_group_name, reference, cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(model_package_group_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference,
cfg.DELETE_WAIT_PERIOD,
cfg.DELETE_WAIT_LENGTH)
assert deleted is True
assert get_sagemaker_model_package_group(
model_package_group_name) is None
def xgboost_churn_data_quality_job_definition(xgboost_churn_endpoint):
endpoint_spec = xgboost_churn_endpoint
endpoint_name = endpoint_spec["spec"].get("endpointName")
resource_name = random_suffix_name("data-quality-job-definition", 32)
replacements = REPLACEMENT_VALUES.copy()
replacements["JOB_DEFINITION_NAME"] = resource_name
replacements["ENDPOINT_NAME"] = endpoint_name
job_definition_reference, _, resource = create_sagemaker_resource(
resource_plural=cfg.DATA_QUALITY_JOB_DEFINITION_RESOURCE_PLURAL,
resource_name=resource_name,
spec_file="data_quality_job_definition_xgboost_churn",
replacements=replacements,
)
assert resource is not None
job_definition_name = resource["spec"].get("jobDefinitionName")
yield (job_definition_reference, resource)
if k8s.get_resource_exists(job_definition_reference):
_, deleted = k8s.delete_custom_resource(job_definition_reference, 3,
10)
assert deleted
def test_create_feature_group(self, feature_group):
"""Tests that a feature group can be created and deleted
using the Feature Group Controller.
"""
(reference, resource) = feature_group
assert k8s.get_resource_exists(reference)
feature_group_name = resource["spec"].get("featureGroupName", None)
assert feature_group_name is not None
feature_group_sm_desc = get_sagemaker_feature_group(feature_group_name)
feature_group_arn = feature_group_sm_desc["FeatureGroupArn"]
assert k8s.get_resource_arn(resource) == feature_group_arn
assert feature_group_sm_desc[
"FeatureGroupStatus"] == FEATURE_GROUP_STATUS_CREATING
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
self._assert_feature_group_status_in_sync(
feature_group_name, reference, FEATURE_GROUP_STATUS_CREATED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(feature_group_arn, resource_tags)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference, WAIT_PERIOD_COUNT,
WAIT_PERIOD_LENGTH)
assert deleted
assert get_sagemaker_feature_group(feature_group_name) is None
def test_completed(self, xgboost_training_job):
(reference, resource) = xgboost_training_job
assert k8s.get_resource_exists(reference)
training_job_name = resource["spec"].get("trainingJobName", None)
assert training_job_name is not None
training_job_desc = get_sagemaker_training_job(training_job_name)
training_job_arn = training_job_desc["TrainingJobArn"]
assert k8s.get_resource_arn(resource) == training_job_arn
assert training_job_desc[
"TrainingJobStatus"] == cfg.JOB_STATUS_INPROGRESS
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
assert_training_status_in_sync(training_job_name, reference,
cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
# model artifact URL is populated
resource = k8s.get_resource(reference)
resource["status"]["modelArtifacts"]["s3ModelArtifacts"] is not None
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(training_job_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference,
cfg.JOB_DELETE_WAIT_PERIODS,
cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted is True
def single_variant_config():
config_resource_name = random_suffix_name("single-variant-config", 32)
model_resource_name = config_resource_name + "-model"
replacements = REPLACEMENT_VALUES.copy()
replacements["CONFIG_NAME"] = config_resource_name
replacements["MODEL_NAME"] = model_resource_name
model_reference, model_spec, model_resource = create_sagemaker_resource(
resource_plural=MODEL_RESOURCE_PLURAL,
resource_name=model_resource_name,
spec_file="xgboost_model",
replacements=replacements,
)
assert model_resource is not None
config_reference, config_spec, config_resource = create_sagemaker_resource(
resource_plural=CONFIG_RESOURCE_PLURAL,
resource_name=config_resource_name,
spec_file="endpoint_config_single_variant",
replacements=replacements,
)
assert config_resource is not None
yield (config_reference, config_resource)
k8s.delete_custom_resource(model_reference)
# Delete the k8s resource if not already deleted by tests
if k8s.get_resource_exists(config_reference):
k8s.delete_custom_resource(config_reference)
def test_stopped(self, xgboost_hpojob):
(reference, resource) = xgboost_hpojob
assert k8s.get_resource_exists(reference)
hpo_job_name = resource["spec"].get("hyperParameterTuningJobName",
None)
assert hpo_job_name is not None
hpo_sm_desc = get_sagemaker_hpo_job(hpo_job_name)
assert (k8s.get_resource_arn(resource) ==
hpo_sm_desc["HyperParameterTuningJobArn"])
assert hpo_sm_desc[
"HyperParameterTuningJobStatus"] == cfg.JOB_STATUS_INPROGRESS
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
self._assert_hpo_status_in_sync(hpo_job_name, reference,
cfg.JOB_STATUS_INPROGRESS)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference,
cfg.JOB_DELETE_WAIT_PERIODS,
cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted is True
hpo_sm_desc = get_sagemaker_hpo_job(hpo_job_name)
assert (hpo_sm_desc["HyperParameterTuningJobStatus"]
in cfg.LIST_JOB_STATUS_STOPPED)
def test_smoke(
self, sagemaker_client, xgboost_churn_model_explainability_job_definition
):
(reference, resource) = xgboost_churn_model_explainability_job_definition
assert k8s.get_resource_exists(reference)
job_definition_name = resource["spec"].get("jobDefinitionName")
job_definition_desc = get_sagemaker_model_explainability_job_definition(
sagemaker_client, job_definition_name
)
job_definition_arn = job_definition_desc["JobDefinitionArn"]
assert k8s.get_resource_arn(resource) == job_definition_arn
# random sleep before we check for tags to reduce test flakyness
time.sleep(cfg.TAG_DELAY_SLEEP)
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(job_definition_arn, resource_tags)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference, 3, 10)
assert deleted
assert (
get_sagemaker_model_explainability_job_definition(
sagemaker_client, job_definition_name
)
is None
)
def xgboost_transformjob(xgboost_model_for_transform):
(transform_resource_name, model_resource_name) = xgboost_model_for_transform
replacements = REPLACEMENT_VALUES.copy()
replacements["MODEL_NAME"] = model_resource_name
replacements["TRANSFORM_JOB_NAME"] = transform_resource_name
reference, _, resource = create_sagemaker_resource(
resource_plural=RESOURCE_PLURAL,
resource_name=transform_resource_name,
spec_file="xgboost_transformjob",
replacements=replacements,
)
assert resource is not None
if k8s.get_resource_arn(resource) is None:
logging.error(
f"ARN for this resource is None, resource status is: {resource['status']}"
)
assert k8s.get_resource_arn(resource) is not None
yield (reference, resource)
if k8s.get_resource_exists(reference):
_, deleted = k8s.delete_custom_resource(reference, cfg.JOB_DELETE_WAIT_PERIODS, cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted
def xgboost_churn_data_quality_monitoring_schedule(
xgboost_churn_data_quality_job_definition,
):
(_, job_definition_resource) = xgboost_churn_data_quality_job_definition
job_definition_name = job_definition_resource["spec"].get("jobDefinitionName")
monitoring_schedule_name = random_suffix_name("monitoring-schedule", 32)
replacements = REPLACEMENT_VALUES.copy()
replacements["MONITORING_SCHEDULE_NAME"] = monitoring_schedule_name
replacements["JOB_DEFINITION_NAME"] = job_definition_name
replacements["MONITORING_TYPE"] = "DataQuality"
reference, spec, resource = create_sagemaker_resource(
resource_plural=RESOURCE_PLURAL,
resource_name=monitoring_schedule_name,
spec_file="monitoring_schedule_base",
replacements=replacements,
)
assert resource is not None
yield (reference, resource, spec)
if k8s.get_resource_exists(reference):
_, deleted = k8s.delete_custom_resource(reference, cfg.DELETE_WAIT_PERIOD, cfg.DELETE_WAIT_LENGTH)
assert deleted
def test_completed(self, xgboost_hpojob):
(reference, resource) = xgboost_hpojob
assert k8s.get_resource_exists(reference)
hpo_job_name = resource["spec"].get("hyperParameterTuningJobName",
None)
assert hpo_job_name is not None
hpo_sm_desc = get_sagemaker_hpo_job(hpo_job_name)
hpo_arn = hpo_sm_desc["HyperParameterTuningJobArn"]
assert k8s.get_resource_arn(resource) == hpo_arn
assert hpo_sm_desc[
"HyperParameterTuningJobStatus"] == cfg.JOB_STATUS_INPROGRESS
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
self._assert_hpo_status_in_sync(hpo_job_name, reference,
cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(hpo_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference,
cfg.JOB_DELETE_WAIT_PERIODS,
cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted is True
def test_completed(self, kmeans_processing_job):
(reference, resource) = kmeans_processing_job
assert k8s.get_resource_exists(reference)
processing_job_name = resource["spec"].get("processingJobName", None)
assert processing_job_name is not None
processing_job_desc = get_sagemaker_processing_job(processing_job_name)
processing_job_arn = processing_job_desc["ProcessingJobArn"]
assert k8s.get_resource_arn(resource) == processing_job_arn
assert processing_job_desc["ProcessingJobStatus"] == cfg.JOB_STATUS_INPROGRESS
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
self._assert_processing_status_in_sync(
processing_job_name, reference, cfg.JOB_STATUS_COMPLETED
)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(processing_job_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference, cfg.JOB_DELETE_WAIT_PERIODS, cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted is True
def xgboost_model_package_group():
resource_name = random_suffix_name("xgboost-model-package-group", 38)
replacements = REPLACEMENT_VALUES.copy()
replacements["MODEL_PACKAGE_GROUP_NAME"] = resource_name
(
model_package_group_reference,
model_package_group_spec,
model_package_group_resource,
) = create_sagemaker_resource(
resource_plural=cfg.MODEL_PACKAGE_GROUP_RESOURCE_PLURAL,
resource_name=resource_name,
spec_file="xgboost_model_package_group",
replacements=replacements,
)
assert model_package_group_resource is not None
if k8s.get_resource_arn(model_package_group_resource) is None:
logging.error(
f"ARN for this resource is None, resource status is: {model_package_group_resource['status']}"
)
assert k8s.get_resource_arn(model_package_group_resource) is not None
yield (model_package_group_reference, model_package_group_resource)
# Delete the k8s resource if not already deleted by tests
if k8s.get_resource_exists(model_package_group_reference):
_, deleted = k8s.delete_custom_resource(model_package_group_reference,
DELETE_WAIT_PERIOD,
DELETE_WAIT_LENGTH)
assert deleted
def test_stopped(self, xgboost_training_job):
(reference, resource) = xgboost_training_job
assert k8s.get_resource_exists(reference)
training_job_name = resource["spec"].get("trainingJobName", None)
assert training_job_name is not None
training_job_desc = get_sagemaker_training_job(training_job_name)
assert k8s.get_resource_arn(
resource) == training_job_desc["TrainingJobArn"]
assert training_job_desc[
"TrainingJobStatus"] == cfg.JOB_STATUS_INPROGRESS
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
assert_training_status_in_sync(training_job_name, reference,
cfg.JOB_STATUS_INPROGRESS)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference,
cfg.JOB_DELETE_WAIT_PERIODS,
cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted is True
training_job_desc = get_sagemaker_training_job(training_job_name)
assert training_job_desc[
"TrainingJobStatus"] in cfg.LIST_JOB_STATUS_STOPPED
def xgboost_training_job():
resource_name = random_suffix_name("xgboost-trainingjob", 32)
replacements = REPLACEMENT_VALUES.copy()
replacements["TRAINING_JOB_NAME"] = resource_name
reference, _, resource = create_sagemaker_resource(
resource_plural=RESOURCE_PLURAL,
resource_name=resource_name,
spec_file="xgboost_trainingjob",
replacements=replacements,
)
assert resource is not None
if k8s.get_resource_arn(resource) is None:
logging.error(
f"ARN for this resource is None, resource status is: {resource['status']}"
)
assert k8s.get_resource_arn(resource) is not None
yield (reference, resource)
if k8s.get_resource_exists(reference):
_, deleted = k8s.delete_custom_resource(reference,
cfg.JOB_DELETE_WAIT_PERIODS,
cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted
def test_crud_authorizer(self, api_resource):
api_ref, api_cr = api_resource
api_id = api_cr['status']['apiID']
test_data = REPLACEMENT_VALUES.copy()
authorizer_name = random_suffix_name("ack-test-authorizer", 25)
test_data['AUTHORIZER_NAME'] = authorizer_name
test_data['AUTHORIZER_TITLE'] = authorizer_name
test_data['API_ID'] = api_id
test_data['AUTHORIZER_URI'] = f'arn:aws:apigateway:{get_region()}:lambda:path/2015-03-31/functions/{get_bootstrap_resources().AuthorizerFunctionArn}/invocations'
authorizer_ref, authorizer_data = helper.authorizer_ref_and_data(authorizer_resource_name=authorizer_name,
replacement_values=test_data)
logging.debug(f"http api authorizer resource. name: {authorizer_name}, data: {authorizer_data}")
# test create
k8s.create_custom_resource(authorizer_ref, authorizer_data)
time.sleep(CREATE_WAIT_AFTER_SECONDS)
assert k8s.wait_on_condition(authorizer_ref, "ACK.ResourceSynced", "True", wait_periods=10)
cr = k8s.get_resource(authorizer_ref)
assert cr is not None
authorizer_id = cr['status']['authorizerID']
# Let's check that the HTTP Api integration appears in Amazon API Gateway
apigw_validator.assert_authorizer_is_present(api_id=api_id, authorizer_id=authorizer_id)
apigw_validator.assert_authorizer_name(
api_id=api_id,
authorizer_id=authorizer_id,
expected_authorizer_name=authorizer_name
)
# test update
updated_authorizer_title = 'updated-' + authorizer_name
test_data['AUTHORIZER_TITLE'] = updated_authorizer_title
updated_authorizer_resource_data = load_apigatewayv2_resource(
"authorizer",
additional_replacements=test_data,
)
logging.debug(f"updated http api authorizer resource: {updated_authorizer_resource_data}")
# Update the k8s resource
k8s.patch_custom_resource(authorizer_ref, updated_authorizer_resource_data)
time.sleep(UPDATE_WAIT_AFTER_SECONDS)
condition.assert_synced(authorizer_ref)
# Let's check that the HTTP Api authorizer appears in Amazon API Gateway with updated title
apigw_validator.assert_authorizer_name(
api_id=api_id,
authorizer_id=authorizer_id,
expected_authorizer_name=updated_authorizer_title
)
# test delete
k8s.delete_custom_resource(authorizer_ref)
time.sleep(DELETE_WAIT_AFTER_SECONDS)
assert not k8s.get_resource_exists(authorizer_ref)
# HTTP Api authorizer should no longer appear in Amazon API Gateway
apigw_validator.assert_authorizer_is_deleted(api_id=api_id, authorizer_id=authorizer_id)
def test_versioned_model_package_completed(
self, xgboost_versioned_model_package):
(reference, spec, resource) = xgboost_versioned_model_package
assert k8s.get_resource_exists(reference)
model_package_group_name = resource["spec"].get(
"modelPackageGroupName")
# Model package name for Versioned Model packages is the ARN of the resource
model_package_name = sagemaker_client().list_model_packages(
ModelPackageGroupName=model_package_group_name
)["ModelPackageSummaryList"][0]["ModelPackageArn"]
model_package_desc = get_sagemaker_model_package(model_package_name)
if k8s.get_resource_arn(resource) is None:
logging.error(
f"ARN for this resource is None, resource status is: {resource['status']}"
)
assert k8s.get_resource_arn(resource) == model_package_name
self._assert_model_package_status_in_sync(model_package_name,
reference,
cfg.JOB_STATUS_INPROGRESS)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
self._assert_model_package_status_in_sync(model_package_name,
reference,
cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
# Update the resource
new_model_approval_status = "Approved"
approval_description = "Approved modelpackage"
spec["spec"]["modelApprovalStatus"] = new_model_approval_status
spec["spec"]["approvalDescription"] = approval_description
resource = k8s.patch_custom_resource(reference, spec)
resource = k8s.wait_resource_consumed_by_controller(reference)
assert resource is not None
self._assert_model_package_status_in_sync(model_package_name,
reference,
cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
model_package_desc = get_sagemaker_model_package(model_package_name)
assert model_package_desc[
"ModelApprovalStatus"] == new_model_approval_status
assert model_package_desc[
"ApprovalDescription"] == approval_description
assert (resource["spec"].get("modelApprovalStatus",
None) == new_model_approval_status)
assert resource["spec"].get("approvalDescription",
None) == approval_description
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference, DELETE_WAIT_PERIOD,
DELETE_WAIT_LENGTH)
assert deleted is True
assert get_sagemaker_model_package(model_package_name) is None
def authorizer_resource(api_resource):
random_suffix = (''.join(
random.choice(string.ascii_lowercase) for _ in range(6)))
authorizer_resource_name = test_resource_values[
'AUTHORIZER_NAME'] + f'-{random_suffix}'
test_resource_values['AUTHORIZER_NAME'] = authorizer_resource_name
authorizer_uri = f'arn:aws:apigateway:{get_region()}:lambda:path/2015-03-31/functions/{get_bootstrap_resources().AuthorizerFunctionArn}/invocations'
test_resource_values["AUTHORIZER_URI"] = authorizer_uri
authorizer_ref, authorizer_data = helper.authorizer_ref_and_data(
authorizer_resource_name=authorizer_resource_name,
replacement_values=test_resource_values)
if k8s.get_resource_exists(authorizer_ref):
raise Exception(
f"expected {authorizer_resource_name} to not exist. Did previous test cleanup?"
)
logging.debug(
f"apigatewayv2 authorizer resource. name: {authorizer_resource_name}, data: {authorizer_data}"
)
k8s.create_custom_resource(authorizer_ref, authorizer_data)
cr = k8s.wait_resource_consumed_by_controller(authorizer_ref)
assert cr is not None
assert k8s.get_resource_exists(authorizer_ref)
authorizer_id = cr['status']['authorizerID']
test_resource_values['AUTHORIZER_ID'] = authorizer_id
# add permissions for apigateway to invoke authorizer lambda
authorizer_arn = "arn:aws:execute-api:{region}:{account}:{api_id}/authorizers/{authorizer_id}".format(
region=get_region(),
account=get_account_id(),
api_id=test_resource_values['API_ID'],
authorizer_id=authorizer_id)
lambda_client = boto3.client("lambda")
lambda_client.add_permission(
FunctionName=get_bootstrap_resources().AuthorizerFunctionName,
StatementId=
f'apigatewayv2-authorizer-invoke-permissions-{random_suffix}',
Action='lambda:InvokeFunction',
Principal='apigateway.amazonaws.com',
SourceArn=authorizer_arn)
yield authorizer_ref, cr
k8s.delete_custom_resource(authorizer_ref)
def test_smoke(self, lambda_client):
resource_name = random_suffix_name("lambda-csc", 24)
resources = get_bootstrap_resources()
logging.debug(resources)
replacements = REPLACEMENT_VALUES.copy()
replacements["AWS_REGION"] = get_region()
replacements["CODE_SIGNING_CONFIG_NAME"] = resource_name
replacements["SIGNING_PROFILE_VERSION_ARN"] = resources.SigningProfileVersionArn
# Load Lambda CR
resource_data = load_lambda_resource(
"code_signing_config",
additional_replacements=replacements,
)
logging.debug(resource_data)
# Create k8s resource
ref = k8s.CustomResourceReference(
CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL,
resource_name, namespace="default",
)
k8s.create_custom_resource(ref, resource_data)
cr = k8s.wait_resource_consumed_by_controller(ref)
assert cr is not None
assert k8s.get_resource_exists(ref)
codeSigningConfigARN = cr['status']['ackResourceMetadata']['arn']
time.sleep(CREATE_WAIT_AFTER_SECONDS)
# Check Lambda code signing config exists
exists = self.code_signing_config_exists(lambda_client, codeSigningConfigARN)
assert exists
# Update cr
cr["spec"]["description"] = "new description"
# Patch k8s resource
k8s.patch_custom_resource(ref, cr)
time.sleep(UPDATE_WAIT_AFTER_SECONDS)
# Check code signing config description
csc = self.get_code_signing_config(lambda_client, codeSigningConfigARN)
assert csc is not None
assert csc["Description"] == "new description"
# Delete k8s resource
_, deleted = k8s.delete_custom_resource(ref)
assert deleted
time.sleep(DELETE_WAIT_AFTER_SECONDS)
# Check Lambda code signing config doesn't exist
exists = self.code_signing_config_exists(lambda_client, codeSigningConfigARN)
assert not exists
def user_profile_fixture(domain_fixture):
(domain_reference, domain_resource, domain_spec) = domain_fixture
assert k8s.get_resource_exists(domain_reference)
domain_id = domain_resource["status"].get("domainID", None)
assert domain_id is not None
assert_domain_status_in_sync(domain_id, domain_reference, "InService")
domain_resource = patch_domain_kernel_instance(
domain_reference, domain_spec, "ml.t3.large"
)
wait_for_status("ml.t3.large", 10, 30, get_domain_kernel_instance, domain_id)
assert_domain_status_in_sync(domain_id, domain_reference, "InService")
resource_name = random_suffix_name("profile", 15)
(
user_profile_reference,
user_profile_resource,
user_profile_spec,
) = apply_user_profile_yaml(resource_name, domain_id)
assert user_profile_resource is not None
if k8s.get_resource_arn(user_profile_resource) is None:
logging.error(
f"ARN for this resource is None, resource status is: {user_profile_resource['status']}"
)
assert k8s.get_resource_arn(user_profile_resource) is not None
yield (
domain_reference,
domain_resource,
domain_spec,
user_profile_reference,
user_profile_resource,
user_profile_spec,
)
if k8s.get_resource_exists(user_profile_reference):
_, deleted = k8s.delete_custom_resource(
user_profile_reference,
cfg.JOB_DELETE_WAIT_PERIODS,
cfg.JOB_DELETE_WAIT_LENGTH,
)
assert deleted
def test_repository_lifecycle_policy(self, ecr_client):
resource_name = random_suffix_name("ecr-repository", 24)
replacements = REPLACEMENT_VALUES.copy()
replacements["REPOSITORY_NAME"] = resource_name
# Load ECR CR
resource_data = load_ecr_resource(
"repository_lifecycle_policy",
additional_replacements=replacements,
)
logging.debug(resource_data)
# Create k8s resource
ref = k8s.CustomResourceReference(
CRD_GROUP,
CRD_VERSION,
RESOURCE_PLURAL,
resource_name,
namespace="default",
)
k8s.create_custom_resource(ref, resource_data)
cr = k8s.wait_resource_consumed_by_controller(ref)
assert cr is not None
assert k8s.get_resource_exists(ref)
time.sleep(CREATE_WAIT_AFTER_SECONDS)
# Check ECR repository exists
repo = self.get_repository(ecr_client, resource_name)
assert repo is not None
# Check ECR repository lifecycle policy exists
lifecycle_policy = self.get_lifecycle_policy(ecr_client, resource_name,
repo["registryId"])
assert lifecycle_policy == LIFECYCLE_POLICY_FILTERING_ON_IMAGE_AGE
# Remove lifecycle policy
cr["spec"]["lifecyclePolicy"] = ""
# Patch k8s resource
k8s.patch_custom_resource(ref, cr)
time.sleep(UPDATE_WAIT_AFTER_SECONDS)
lifecycle_policy = self.get_lifecycle_policy(ecr_client, resource_name,
repo["registryId"])
assert lifecycle_policy == ""
# Delete k8s resource
_, deleted = k8s.delete_custom_resource(ref)
assert deleted is True
time.sleep(DELETE_WAIT_AFTER_SECONDS)
# Check ECR repository doesn't exists
exists = self.repository_exists(ecr_client, resource_name)
assert not exists
def test_smoke(self, lambda_client, lambda_function):
(_, function_resource) = lambda_function
lambda_function_name = function_resource["spec"]["name"]
resource_name = random_suffix_name("lambda-alias", 24)
replacements = REPLACEMENT_VALUES.copy()
replacements["AWS_REGION"] = get_region()
replacements["ALIAS_NAME"] = resource_name
replacements["FUNCTION_NAME"] = lambda_function_name
replacements["FUNCTION_VERSION"] = "$LATEST"
# Load alias CR
resource_data = load_lambda_resource(
"alias",
additional_replacements=replacements,
)
logging.debug(resource_data)
# Create k8s resource
ref = k8s.CustomResourceReference(
CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL,
resource_name, namespace="default",
)
k8s.create_custom_resource(ref, resource_data)
cr = k8s.wait_resource_consumed_by_controller(ref)
assert cr is not None
assert k8s.get_resource_exists(ref)
time.sleep(CREATE_WAIT_AFTER_SECONDS)
# Check alias exists
alias = self.alias_exist(lambda_client, resource_name, lambda_function_name)
assert alias is not None
# Update cr
cr["spec"]["description"] = ""
# Patch k8s resource
k8s.patch_custom_resource(ref, cr)
time.sleep(UPDATE_WAIT_AFTER_SECONDS)
# Check alias description
alias = self.get_alias(lambda_client, resource_name, lambda_function_name)
assert alias is not None
assert alias["Description"] == ""
# Delete k8s resource
_, deleted = k8s.delete_custom_resource(ref)
assert deleted
time.sleep(DELETE_WAIT_AFTER_SECONDS)
# Check alias doesn't exist
exists = self.get_alias(lambda_client, resource_name, lambda_function_name)
assert not exists
def test_user_password(self, user_password, user_password_input):
(reference, resource) = user_password
assert k8s.get_resource_exists(reference)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=5)
resource = k8s.get_resource(reference)
assert resource["status"]["authentication"] is not None
assert resource["status"]["authentication"]["type_"] == "password"
assert resource["status"]["authentication"]["passwordCount"] == 2