def update_endpoint_failed_test(self, sagemaker_client,
single_variant_config, faulty_config,
xgboost_endpoint):
(endpoint_reference, _, endpoint_spec) = xgboost_endpoint
(_, faulty_config_resource) = faulty_config
faulty_config_name = faulty_config_resource["spec"].get(
"endpointConfigName", None)
endpoint_spec["spec"]["endpointConfigName"] = faulty_config_name
endpoint_resource = k8s.patch_custom_resource(endpoint_reference,
endpoint_spec)
endpoint_resource = k8s.wait_resource_consumed_by_controller(
endpoint_reference)
assert endpoint_resource is not None
# endpoint transitions Updating -> InService state
self._assert_endpoint_status_in_sync(
sagemaker_client,
endpoint_reference.name,
endpoint_reference,
self.status_udpating,
)
assert k8s.wait_on_condition(endpoint_reference, "ACK.ResourceSynced",
"False")
endpoint_resource = k8s.get_resource(endpoint_reference)
assert (endpoint_resource["status"].get(
"lastEndpointConfigNameForUpdate", None) == faulty_config_name)
self._assert_endpoint_status_in_sync(
sagemaker_client,
endpoint_reference.name,
endpoint_reference,
self.status_inservice,
)
assert k8s.wait_on_condition(endpoint_reference, "ACK.ResourceSynced",
"True")
assert k8s.assert_condition_state_message(
endpoint_reference,
"ACK.Terminal",
"True",
"Unable to update Endpoint. Check FailureReason",
)
endpoint_resource = k8s.get_resource(endpoint_reference)
assert endpoint_resource["status"].get("failureReason",
None) is not None
# additional check: endpoint using old endpoint config
(_, old_config_resource) = single_variant_config
current_config_name = endpoint_resource["status"].get(
"latestEndpointConfigName")
assert (current_config_name is not None
and current_config_name == old_config_resource["spec"].get(
"endpointConfigName", None))
def update_endpoint_failed_test(self, single_variant_config, faulty_config,
xgboost_endpoint):
(endpoint_reference, _, endpoint_spec) = xgboost_endpoint
(_, faulty_config_resource) = faulty_config
faulty_config_name = faulty_config_resource["spec"].get(
"endpointConfigName", None)
endpoint_spec["spec"]["endpointConfigName"] = faulty_config_name
endpoint_resource = k8s.patch_custom_resource(endpoint_reference,
endpoint_spec)
endpoint_resource = k8s.wait_resource_consumed_by_controller(
endpoint_reference)
assert endpoint_resource is not None
# endpoint transitions Updating -> InService state
assert_endpoint_status_in_sync(
endpoint_reference.name,
endpoint_reference,
cfg.ENDPOINT_STATUS_UPDATING,
)
assert k8s.wait_on_condition(endpoint_reference, "ACK.ResourceSynced",
"False")
endpoint_resource = k8s.get_resource(endpoint_reference)
annotations = endpoint_resource["metadata"].get("annotations", None)
assert annotations is not None
assert annotations[
LAST_ENDPOINTCONFIG_UPDATE_ANNOTATION] == faulty_config_name
assert_endpoint_status_in_sync(
endpoint_reference.name,
endpoint_reference,
cfg.ENDPOINT_STATUS_INSERVICE,
)
assert k8s.wait_on_condition(endpoint_reference, "ACK.ResourceSynced",
"False")
(_, old_config_resource) = single_variant_config
current_config_name = old_config_resource["spec"].get(
"endpointConfigName", None)
assert k8s.assert_condition_state_message(
endpoint_reference,
"ACK.Terminal",
"True",
FAIL_UPDATE_ERROR_MESSAGE + current_config_name,
)
endpoint_resource = k8s.get_resource(endpoint_reference)
assert endpoint_resource["status"].get("failureReason",
None) is not None
def get_training_rule_eval_resource_status(
reference: k8s.CustomResourceReference, rule_type: str):
resource = k8s.get_resource(reference)
resource_status = resource["status"][
rule_type + "EvaluationStatuses"][0]["ruleEvaluationStatus"]
assert resource_status is not None
return resource_status
def test_user_nopass(self, user_nopass, user_nopass_input):
(reference, resource) = user_nopass
assert k8s.get_resource_exists(reference)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=5)
resource = k8s.get_resource(reference)
assert resource["status"]["lastRequestedAccessString"] == user_nopass_input["ACCESS_STRING"]
new_access_string = "on ~app::* -@all +@read +@write"
user_patch = {"spec": {"accessString": new_access_string}}
_ = k8s.patch_custom_resource(reference, user_patch)
sleep(DEFAULT_WAIT_SECS)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=5)
resource = k8s.get_resource(reference)
assert resource["status"]["lastRequestedAccessString"] == new_access_string
def test_completed(self, xgboost_training_job):
(reference, resource) = xgboost_training_job
assert k8s.get_resource_exists(reference)
training_job_name = resource["spec"].get("trainingJobName", None)
assert training_job_name is not None
training_job_desc = get_sagemaker_training_job(training_job_name)
training_job_arn = training_job_desc["TrainingJobArn"]
assert k8s.get_resource_arn(resource) == training_job_arn
assert training_job_desc[
"TrainingJobStatus"] == cfg.JOB_STATUS_INPROGRESS
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False")
assert_training_status_in_sync(training_job_name, reference,
cfg.JOB_STATUS_COMPLETED)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
# model artifact URL is populated
resource = k8s.get_resource(reference)
resource["status"]["modelArtifacts"]["s3ModelArtifacts"] is not None
resource_tags = resource["spec"].get("tags", None)
assert_tags_in_sync(training_job_arn, resource_tags)
# Check that you can delete a completed resource from k8s
_, deleted = k8s.delete_custom_resource(reference,
cfg.JOB_DELETE_WAIT_PERIODS,
cfg.JOB_DELETE_WAIT_LENGTH)
assert deleted is True
def test_rg_upgrade_ev(self, rg_upgrade_ev_input, rg_upgrade_ev):
(reference, _) = rg_upgrade_ev
assert k8s.wait_on_condition(reference,
"ACK.ResourceSynced",
"True",
wait_periods=90)
# assert initial state
cc = retrieve_cache_cluster(rg_upgrade_ev_input['RG_ID'])
assert cc is not None
assert cc['EngineVersion'] == rg_upgrade_ev_input['ENGINE_VERSION']
# upgrade engine version, wait for resource to sync
desired_engine_version = "5.0.6"
patch = {"spec": {"engineVersion": desired_engine_version}}
_ = k8s.patch_custom_resource(reference, patch)
sleep(DEFAULT_WAIT_SECS)
assert k8s.wait_on_condition(reference,
"ACK.ResourceSynced",
"True",
wait_periods=90)
# assert new state after upgrading engine version
resource = k8s.get_resource(reference)
assert resource['status']['status'] == "available"
assert resource['spec']['engineVersion'] == desired_engine_version
cc = retrieve_cache_cluster(rg_upgrade_ev_input['RG_ID'])
assert cc is not None
assert cc['EngineVersion'] == desired_engine_version
def test_crud_authorizer(self, api_resource):
api_ref, api_cr = api_resource
api_id = api_cr['status']['apiID']
test_data = REPLACEMENT_VALUES.copy()
authorizer_name = random_suffix_name("ack-test-authorizer", 25)
test_data['AUTHORIZER_NAME'] = authorizer_name
test_data['AUTHORIZER_TITLE'] = authorizer_name
test_data['API_ID'] = api_id
test_data['AUTHORIZER_URI'] = f'arn:aws:apigateway:{get_region()}:lambda:path/2015-03-31/functions/{get_bootstrap_resources().AuthorizerFunctionArn}/invocations'
authorizer_ref, authorizer_data = helper.authorizer_ref_and_data(authorizer_resource_name=authorizer_name,
replacement_values=test_data)
logging.debug(f"http api authorizer resource. name: {authorizer_name}, data: {authorizer_data}")
# test create
k8s.create_custom_resource(authorizer_ref, authorizer_data)
time.sleep(CREATE_WAIT_AFTER_SECONDS)
assert k8s.wait_on_condition(authorizer_ref, "ACK.ResourceSynced", "True", wait_periods=10)
cr = k8s.get_resource(authorizer_ref)
assert cr is not None
authorizer_id = cr['status']['authorizerID']
# Let's check that the HTTP Api integration appears in Amazon API Gateway
apigw_validator.assert_authorizer_is_present(api_id=api_id, authorizer_id=authorizer_id)
apigw_validator.assert_authorizer_name(
api_id=api_id,
authorizer_id=authorizer_id,
expected_authorizer_name=authorizer_name
)
# test update
updated_authorizer_title = 'updated-' + authorizer_name
test_data['AUTHORIZER_TITLE'] = updated_authorizer_title
updated_authorizer_resource_data = load_apigatewayv2_resource(
"authorizer",
additional_replacements=test_data,
)
logging.debug(f"updated http api authorizer resource: {updated_authorizer_resource_data}")
# Update the k8s resource
k8s.patch_custom_resource(authorizer_ref, updated_authorizer_resource_data)
time.sleep(UPDATE_WAIT_AFTER_SECONDS)
condition.assert_synced(authorizer_ref)
# Let's check that the HTTP Api authorizer appears in Amazon API Gateway with updated title
apigw_validator.assert_authorizer_name(
api_id=api_id,
authorizer_id=authorizer_id,
expected_authorizer_name=updated_authorizer_title
)
# test delete
k8s.delete_custom_resource(authorizer_ref)
time.sleep(DELETE_WAIT_AFTER_SECONDS)
assert not k8s.get_resource_exists(authorizer_ref)
# HTTP Api authorizer should no longer appear in Amazon API Gateway
apigw_validator.assert_authorizer_is_deleted(api_id=api_id, authorizer_id=authorizer_id)
def update_notebook_test(self, notebook_instance):
(reference, resource, spec) = notebook_instance
notebook_instance_name = resource["spec"].get("notebookInstanceName",
None)
volumeSizeInGB = 7
additionalCodeRepositories = [
"https://github.com/aws-controllers-k8s/runtime"
]
spec["spec"]["volumeSizeInGB"] = volumeSizeInGB
# TODO: Use del spec["spec"]["defaultCodeRepository"] instead once ack.testinfra supports replacement.
# Patch only supports updating spec fields instead of fully getting rid of them.
spec["spec"]["defaultCodeRepository"] = None
spec["spec"]["additionalCodeRepositories"] = additionalCodeRepositories
k8s.patch_custom_resource(reference, spec)
self._assert_notebook_status_in_sync(notebook_instance_name, reference,
"Stopping")
# TODO: Replace with annotations once runtime can update annotations in readOne.
resource = k8s.get_resource(reference)
# Test is flakey as this field can get changed before we get resource
# UpdateTriggered can only be in the status if beforehand it was UpdatePending
# TODO: See if update code can be restructured to avoid this
assert resource["status"]["stoppedByControllerMetadata"] in (
"UpdatePending", "UpdateTriggered")
# wait for the resource to go to the InService state and make sure the operator is synced with sagemaker.
self._assert_notebook_status_in_sync(notebook_instance_name, reference,
"InService")
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True")
notebook_instance_desc = get_notebook_instance(notebook_instance_name)
assert notebook_instance_desc["VolumeSizeInGB"] == volumeSizeInGB
resource = k8s.get_resource(reference)
assert resource["spec"]["volumeSizeInGB"] == volumeSizeInGB
assert "DefaultCodeRepository" not in notebook_instance_desc
assert "defaultCodeRepository" not in resource["spec"]
assert resource["spec"][
"additionalCodeRepositories"] == additionalCodeRepositories
assert notebook_instance_desc[
"AdditionalCodeRepositories"] == additionalCodeRepositories
assert "stoppedByControllerMetadata" not in resource["status"]
def assert_misc_fields(reference, rg_id, pmw, description, srl, sw):
resource = k8s.get_resource(reference)
cc = retrieve_cache_cluster(rg_id)
rg = retrieve_replication_group(rg_id)
assert cc is not None
assert cc['PreferredMaintenanceWindow'] == pmw
assert resource['spec']['description'] == description
assert rg['SnapshotRetentionLimit'] == srl
assert rg['SnapshotWindow'] == sw
def test_user_password(self, user_password, user_password_input):
(reference, resource) = user_password
assert k8s.get_resource_exists(reference)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=5)
resource = k8s.get_resource(reference)
assert resource["status"]["authentication"] is not None
assert resource["status"]["authentication"]["type_"] == "password"
assert resource["status"]["authentication"]["passwordCount"] == 2
def test_crud_integration(self, api_resource):
api_ref, api_cr = api_resource
api_id = api_cr['status']['apiID']
test_data = REPLACEMENT_VALUES.copy()
integration_name = random_suffix_name("ack-test-integration", 25)
test_data['INTEGRATION_NAME'] = integration_name
test_data['API_ID'] = api_id
integration_ref, integration_data = helper.integration_ref_and_data(integration_resource_name=integration_name,
replacement_values=test_data)
logging.debug(f"http api integration resource. name: {integration_name}, data: {integration_data}")
# test create
k8s.create_custom_resource(integration_ref, integration_data)
time.sleep(CREATE_WAIT_AFTER_SECONDS)
assert k8s.wait_on_condition(integration_ref, "ACK.ResourceSynced", "True", wait_periods=10)
cr = k8s.get_resource(integration_ref)
assert cr is not None
integration_id = cr['status']['integrationID']
# Let's check that the HTTP Api integration appears in Amazon API Gateway
apigw_validator.assert_integration_is_present(api_id=api_id, integration_id=integration_id)
apigw_validator.assert_integration_uri(
api_id=api_id,
integration_id=integration_id,
expected_uri=test_data['INTEGRATION_URI']
)
# test update
updated_uri = 'https://httpbin.org/post'
test_data['INTEGRATION_URI'] = updated_uri
updated_integration_resource_data = load_apigatewayv2_resource(
"integration",
additional_replacements=test_data,
)
logging.debug(f"updated http api integration resource: {updated_integration_resource_data}")
# Update the k8s resource
k8s.patch_custom_resource(integration_ref, updated_integration_resource_data)
time.sleep(UPDATE_WAIT_AFTER_SECONDS)
condition.assert_synced(integration_ref)
# Let's check that the HTTP Api integration appears in Amazon API Gateway with updated uri
apigw_validator.assert_integration_uri(
api_id=api_id,
integration_id=integration_id,
expected_uri=updated_uri
)
# test delete
k8s.delete_custom_resource(integration_ref)
time.sleep(DELETE_WAIT_AFTER_SECONDS)
assert not k8s.get_resource_exists(integration_ref)
# HTTP Api integration should no longer appear in Amazon API Gateway
apigw_validator.assert_integration_is_deleted(api_id=api_id, integration_id=integration_id)
def test_rg_deletion(self, rg_deletion_input, rg_deletion, rg_deletion_waiter):
(reference, _) = rg_deletion
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
# assertions after initial creation
resource = k8s.get_resource(reference)
assert resource['status']['status'] == "available"
# delete
k8s.delete_custom_resource(reference)
sleep(DEFAULT_WAIT_SECS)
resource = k8s.get_resource(reference)
assert resource['metadata']['deletionTimestamp'] is not None
# TODO: uncomment when reconciler->cleanup() invokes patchResource()
# assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False", wait_periods=1)
rg_deletion_waiter.wait(ReplicationGroupId=rg_deletion_input["RG_ID"])
def test_rg_largecluster(self, rg_largecluster_input, rg_largecluster):
(reference, _) = rg_largecluster
assert k8s.wait_on_condition(reference,
"ACK.ResourceSynced",
"True",
wait_periods=240)
# assertions after initial creation
desired_node_groups = int(rg_largecluster_input['NUM_NODE_GROUPS'])
desired_replica_count = int(
rg_largecluster_input['REPLICAS_PER_NODE_GROUP'])
desired_total_nodes = (desired_node_groups *
(1 + desired_replica_count))
resource = k8s.get_resource(reference)
assert resource['status']['status'] == "available"
assert len(resource['status']['nodeGroups']) == desired_node_groups
assert len(resource['status']['memberClusters']) == desired_total_nodes
# update, wait for resource to sync
desired_node_groups = desired_node_groups - 10
desired_total_nodes = (desired_node_groups *
(1 + desired_replica_count))
patch = {
"spec": {
"numNodeGroups":
desired_node_groups,
"nodeGroupConfiguration":
provide_node_group_configuration(desired_node_groups)
}
}
_ = k8s.patch_custom_resource(reference, patch)
sleep(
DEFAULT_WAIT_SECS
) # required as controller has likely not placed the resource in modifying
assert k8s.wait_on_condition(reference,
"ACK.ResourceSynced",
"True",
wait_periods=240)
# assert new state after scaling in
resource = k8s.get_resource(reference)
assert resource['status']['status'] == "available"
assert len(resource['status']['nodeGroups']) == desired_node_groups
assert len(resource['status']['memberClusters']) == desired_total_nodes
def test_rg_cme_even_shards(self, rg_cme_even_shards,
rg_cme_even_shards_input):
(reference, _) = rg_cme_even_shards
assert k8s.wait_on_condition(reference,
"ACK.ResourceSynced",
"True",
wait_periods=90)
nng = int(rg_cme_even_shards_input['NUM_NODE_GROUPS'])
rpng = int(rg_cme_even_shards_input['REPLICAS_PER_NODE_GROUP'])
# assert initial state
resource = k8s.get_resource(reference)
assert len(resource['status']['nodeGroups']) == nng
assert_even_shards_replica_count(resource, rpng)
# increase replica count, wait for resource to sync
rpng += 1
patch = {"spec": {"replicasPerNodeGroup": rpng}}
_ = k8s.patch_custom_resource(reference, patch)
sleep(DEFAULT_WAIT_SECS)
assert k8s.wait_on_condition(reference,
"ACK.ResourceSynced",
"True",
wait_periods=90)
# assert replica count has increased
resource = k8s.get_resource(reference)
assert len(resource['status']['nodeGroups']) == nng
assert_even_shards_replica_count(resource, rpng)
# decrease replica count, wait for resource to sync
rpng -= 2
patch = {"spec": {"replicasPerNodeGroup": rpng}}
_ = k8s.patch_custom_resource(reference, patch)
sleep(DEFAULT_WAIT_SECS)
assert k8s.wait_on_condition(reference,
"ACK.ResourceSynced",
"True",
wait_periods=90)
# assert replica count has decreased
resource = k8s.get_resource(reference)
assert len(resource['status']['nodeGroups']) == nng
assert_even_shards_replica_count(resource, rpng)
def test_rg_cmd_update(self, rg_cmd_update_input, rg_cmd_update):
(reference, _) = rg_cmd_update
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
# assertions after initial creation
desired_node_groups = int(rg_cmd_update_input['NUM_NODE_GROUPS'])
desired_replica_count = int(rg_cmd_update_input['REPLICAS_PER_NODE_GROUP'])
desired_total_nodes = (desired_node_groups * (1 + desired_replica_count))
resource = k8s.get_resource(reference)
assert resource['status']['status'] == "available"
assert len(resource['status']['nodeGroups']) == desired_node_groups
assert len(resource['status']['memberClusters']) == desired_total_nodes
cc = retrieve_cache_cluster(rg_cmd_update_input['RG_ID'])
assert cc is not None
assert cc['EngineVersion'] == rg_cmd_update_input['ENGINE_VERSION']
# increase replica count, wait for resource to sync
desired_replica_count += 1
desired_total_nodes = (desired_node_groups * (1 + desired_replica_count))
patch = {"spec": {"replicasPerNodeGroup": desired_replica_count}}
_ = k8s.patch_custom_resource(reference, patch)
sleep(DEFAULT_WAIT_SECS) # required as controller has likely not placed the resource in modifying
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
# assert new state after increasing replica count
resource = k8s.get_resource(reference)
assert resource['status']['status'] == "available"
assert len(resource['status']['nodeGroups']) == desired_node_groups
assert len(resource['status']['memberClusters']) == desired_total_nodes
# upgrade engine version, wait for resource to sync
desired_engine_version = "5.0.6"
patch = {"spec": {"engineVersion": desired_engine_version}}
_ = k8s.patch_custom_resource(reference, patch)
sleep(DEFAULT_WAIT_SECS)
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
# assert new state after upgrading engine version
resource = k8s.get_resource(reference)
assert resource['status']['status'] == "available"
assert resource['spec']['engineVersion'] == desired_engine_version
cc = retrieve_cache_cluster(rg_cmd_update_input['RG_ID'])
assert cc is not None
assert cc['EngineVersion'] == desired_engine_version
def test_crud_stage(self, api_resource):
api_ref, api_cr = api_resource
api_id = api_cr['status']['apiID']
test_data = REPLACEMENT_VALUES.copy()
stage_name = random_suffix_name("ack-test-stage", 25)
test_data['STAGE_NAME'] = stage_name
test_data['API_ID'] = api_id
stage_ref, stage_data = helper.stage_ref_and_data(stage_resource_name=stage_name,
replacement_values=test_data)
logging.debug(f"http api stage resource. name: {stage_name}, data: {stage_data}")
# test create
k8s.create_custom_resource(stage_ref, stage_data)
time.sleep(CREATE_WAIT_AFTER_SECONDS)
assert k8s.wait_on_condition(stage_ref, "ACK.ResourceSynced", "True", wait_periods=10)
cr = k8s.get_resource(stage_ref)
assert cr is not None
# Let's check that the HTTP Api integration appears in Amazon API Gateway
apigw_validator.assert_stage_is_present(api_id=api_id, stage_name=stage_name)
stage_description = test_data['STAGE_DESCRIPTION']
apigw_validator.assert_stage_description(
api_id=api_id,
stage_name=stage_name,
expected_description=stage_description
)
# test update
updated_description = 'updated' + stage_description
test_data['STAGE_DESCRIPTION'] = updated_description
updated_stage_resource_data = load_apigatewayv2_resource(
"stage",
additional_replacements=test_data,
)
logging.debug(f"updated http api stage resource: {updated_stage_resource_data}")
# Update the k8s resource
k8s.patch_custom_resource(stage_ref, updated_stage_resource_data)
time.sleep(UPDATE_WAIT_AFTER_SECONDS)
condition.assert_synced(stage_ref)
# Let's check that the HTTP Api stage appears in Amazon API Gateway with updated description
apigw_validator.assert_stage_description(
api_id=api_id,
stage_name=stage_name,
expected_description=updated_description
)
# test delete
k8s.delete_custom_resource(stage_ref)
time.sleep(DELETE_WAIT_AFTER_SECONDS)
assert not k8s.get_resource_exists(stage_ref)
# HTTP Api stage should no longer appear in Amazon API Gateway
apigw_validator.assert_stage_is_deleted(api_id=api_id, stage_name=stage_name)
def test_model_has_correct_arn(self, sagemaker_client, xgboost_model):
(reference, _) = xgboost_model
resource = k8s.get_resource(reference)
model_name = resource["spec"].get("modelName", None)
assert model_name is not None
assert self._get_resource_model_arn(resource) == self._get_sagemaker_model_arn(
sagemaker_client, model_name
)
def test_model_is_deleted(self, sagemaker_client, xgboost_model):
(reference, _) = xgboost_model
resource = k8s.get_resource(reference)
model_name = resource["spec"].get("modelName", None)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference)
assert deleted is True
assert self._get_sagemaker_model_arn(sagemaker_client, model_name) is None
def test_config_is_deleted(self, sagemaker_client, single_variant_config):
(reference, _) = single_variant_config
resource = k8s.get_resource(reference)
config_name = resource["spec"].get("endpointConfigName", None)
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference)
assert deleted is True
assert (self._get_sagemaker_endpoint_config_arn(
sagemaker_client, config_name) is None)
def test_config_has_correct_arn(self, sagemaker_client,
single_variant_config):
(reference, _) = single_variant_config
resource = k8s.get_resource(reference)
config_name = resource["spec"].get("endpointConfigName", None)
assert config_name is not None
assert self._get_resource_endpoint_config_arn(
resource) == self._get_sagemaker_endpoint_config_arn(
sagemaker_client, config_name)
def test_crud_httpapi_using_import(self):
test_data = REPLACEMENT_VALUES.copy()
api_name = random_suffix_name("ack-test-importapi", 25)
test_data['IMPORT_API_NAME'] = api_name
test_data['IMPORT_API_TITLE'] = api_name
api_ref, api_data = helper.import_api_ref_and_data(api_resource_name=api_name,
replacement_values=test_data)
logging.debug(f"imported http api resource. name: {api_name}, data: {api_data}")
# test create
k8s.create_custom_resource(api_ref, api_data)
time.sleep(CREATE_API_WAIT_AFTER_SECONDS)
assert k8s.wait_on_condition(api_ref, "ACK.ResourceSynced", "True", wait_periods=10)
cr = k8s.get_resource(api_ref)
assert cr is not None
api_id = cr['status']['apiID']
# Let's check that the imported HTTP Api appears in Amazon API Gateway
apigw_validator.assert_api_is_present(api_id=api_id)
apigw_validator.assert_api_name(
api_id=api_id,
expected_api_name=api_name
)
# test update
updated_api_title = 'updated-' + api_name
test_data['IMPORT_API_TITLE'] = updated_api_title
updated_api_resource_data = load_apigatewayv2_resource(
"import_api",
additional_replacements=test_data,
)
logging.debug(f"updated import http api resource: {updated_api_resource_data}")
# Update the k8s resource
k8s.patch_custom_resource(api_ref, updated_api_resource_data)
time.sleep(UPDATE_WAIT_AFTER_SECONDS)
condition.assert_synced(api_ref)
# Let's check that the HTTP Api appears in Amazon API Gateway with updated title
apigw_validator.assert_api_name(
api_id=api_id,
expected_api_name=updated_api_title
)
# test delete
k8s.delete_custom_resource(api_ref)
time.sleep(DELETE_WAIT_AFTER_SECONDS)
assert not k8s.get_resource_exists(api_ref)
# HTTP Api should no longer appear in Amazon API Gateway
apigw_validator.assert_api_is_deleted(api_id=api_id)
def test_trainingjob_has_created_status(self, sagemaker_client,
xgboost_trainingjob):
(reference, _) = xgboost_trainingjob
resource = k8s.get_resource(reference)
trainingjob_name = resource["spec"].get("trainingJobName", None)
assert trainingjob_name is not None
current_trainingjob_status = self._get_sagemaker_trainingjob_status(
sagemaker_client, trainingjob_name)
expected_trainingjob_status_list = self._get_created_trainingjob_status_list(
)
assert current_trainingjob_status in expected_trainingjob_status_list
def test_processing_job_has_created_status(self, sagemaker_client,
kmeans_processing_job):
(reference, _) = kmeans_processing_job
resource = k8s.get_resource(reference)
processing_job_name = resource["spec"].get("processingJobName", None)
assert processing_job_name is not None
current_processing_job_status = self._get_sagemaker_processing_job_status(
sagemaker_client, processing_job_name)
expected_processing_job_status_list = (
self._get_created_processing_job_status_list())
assert current_processing_job_status in expected_processing_job_status_list
def test_trainingjob_has_correct_arn(self, sagemaker_client,
xgboost_trainingjob):
(reference, _) = xgboost_trainingjob
resource = k8s.get_resource(reference)
trainingjob_name = resource["spec"].get("trainingJobName", None)
assert trainingjob_name is not None
resource_trainingjob_arn = k8s.get_resource_arn(resource)
expected_trainingjob_arn = self._get_sagemaker_trainingjob_arn(
sagemaker_client, trainingjob_name)
assert resource_trainingjob_arn == expected_trainingjob_arn
def test_processing_job_has_correct_arn(self, sagemaker_client,
kmeans_processing_job):
(reference, _) = kmeans_processing_job
resource = k8s.get_resource(reference)
processing_job_name = resource["spec"].get("processingJobName", None)
assert processing_job_name is not None
resource_processing_job_arn = k8s.get_resource_arn(resource)
expected_processing_job_arn = self._get_sagemaker_processing_job_arn(
sagemaker_client, processing_job_name)
assert resource_processing_job_arn == expected_processing_job_arn
def assert_log_delivery_config(reference, config):
resource = k8s.get_resource(reference)
# if log delivery is disabled, logDeliveryConfigurations should be empty or none
if not config['enabled']:
assert 'logDeliveryConfigurations' not in resource['status']
else:
latest = resource['status']['logDeliveryConfigurations'][0]
assert latest['status'] == "active"
assert latest['destinationDetails'] == config['destinationDetails']
assert latest['destinationType'] == config['destinationType']
assert latest['logFormat'] == config['logFormat']
assert latest['logType'] == config['logType']
def assert_expectations(self, verb: str, input_data: dict,
expectations: model.ExpectDict,
reference: k8s.CustomResourceReference) -> None:
"""Asserts custom resource reference inside Kubernetes cluster against the supplied expectations
:param verb: expectations after performing the verb (apply, patch, delete)
:param input_data: input data to verb
:param expectations: expectations to assert
:param reference: custom resource reference
:return: None
"""
self._assert_conditions(expectations, reference, wait=False)
# conditions expectations met, now check current resource against expectations
resource = k8s.get_resource(reference)
self.assert_items(expectations.get("status"), resource.get("status"))
def test_trainingjob_has_stopped_status(self, sagemaker_client,
xgboost_trainingjob):
(reference, _) = xgboost_trainingjob
resource = k8s.get_resource(reference)
trainingjob_name = resource["spec"].get("trainingJobName", None)
assert trainingjob_name is not None
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference)
assert deleted is True
current_trainingjob_status = self._get_sagemaker_trainingjob_status(
sagemaker_client, trainingjob_name)
expected_trainingjob_status_list = self._get_stopped_trainingjob_status_list(
)
assert current_trainingjob_status in expected_trainingjob_status_list
def test_processing_job_has_stopped_status(self, sagemaker_client,
kmeans_processing_job):
(reference, _) = kmeans_processing_job
resource = k8s.get_resource(reference)
processing_job_name = resource["spec"].get("processingJobName", None)
assert processing_job_name is not None
# Delete the k8s resource.
_, deleted = k8s.delete_custom_resource(reference)
assert deleted is True
current_processing_job_status = self._get_sagemaker_processing_job_status(
sagemaker_client, processing_job_name)
expected_processing_job_status_list = (
self._get_stopped_processing_job_status_list())
assert current_processing_job_status in expected_processing_job_status_list
def faulty_config(name_suffix, single_container_model):
replacements = REPLACEMENT_VALUES.copy()
# copy model data to a temp S3 location and delete it after model is created on SageMaker
model_bucket = replacements["SAGEMAKER_DATA_BUCKET"]
copy_source = {
"Bucket": model_bucket,
"Key": "sagemaker/model/xgboost-mnist-model.tar.gz",
}
model_destination_key = "sagemaker/model/delete/xgboost-mnist-model.tar.gz"
s3.copy_object(model_bucket, copy_source, model_destination_key)
model_resource_name = name_suffix + "faulty-model"
replacements["MODEL_NAME"] = model_resource_name
replacements[
"MODEL_LOCATION"] = f"s3://{model_bucket}/{model_destination_key}"
model_reference, model_spec, model_resource = create_sagemaker_resource(
resource_plural=MODEL_RESOURCE_PLURAL,
resource_name=model_resource_name,
spec_file="xgboost_model_with_model_location",
replacements=replacements,
)
assert model_resource is not None
model_resource = k8s.get_resource(model_reference)
assert ("ackResourceMetadata" in model_resource["status"]
and "arn" in model_resource["status"]["ackResourceMetadata"])
s3.delete_object(model_bucket, model_destination_key)
config_resource_name = name_suffix + "-faulty-config"
(_, model_resource) = single_container_model
model_resource_name = model_resource["spec"].get("modelName", None)
replacements["CONFIG_NAME"] = config_resource_name
config_reference, config_spec, config_resource = create_sagemaker_resource(
resource_plural=CONFIG_RESOURCE_PLURAL,
resource_name=config_resource_name,
spec_file="endpoint_config_multi_variant",
replacements=replacements,
)
assert config_resource is not None
yield (config_reference, config_resource)
k8s.delete_custom_resource(model_reference)
k8s.delete_custom_resource(config_reference)