def route_resource(integration_resource, authorizer_resource): random_suffix = (''.join( random.choice(string.ascii_lowercase) for _ in range(6))) route_resource_name = test_resource_values[ 'ROUTE_NAME'] + f'-{random_suffix}' test_resource_values['ROUTE_NAME'] = route_resource_name route_ref, route_data = helper.route_ref_and_data( route_resource_name=route_resource_name, replacement_values=test_resource_values) if k8s.get_resource_exists(route_ref): raise Exception( f"expected {route_resource_name} to not exist. Did previous test cleanup?" ) logging.debug( f"apigatewayv2 route resource. name: {route_resource_name}, data: {route_data}" ) k8s.create_custom_resource(route_ref, route_data) cr = k8s.wait_resource_consumed_by_controller(route_ref) assert cr is not None assert k8s.get_resource_exists(route_ref) route_id = cr['status']['routeID'] test_resource_values['ROUTE_ID'] = route_id yield route_ref, cr k8s.delete_custom_resource(route_ref)
def api_resource(): random_suffix = (''.join( random.choice(string.ascii_lowercase) for _ in range(6))) api_resource_name = test_resource_values['API_NAME'] + f'-{random_suffix}' test_resource_values['API_NAME'] = api_resource_name api_ref, api_data = helper.api_ref_and_data( api_resource_name=api_resource_name, replacement_values=test_resource_values) if k8s.get_resource_exists(api_ref): raise Exception( f"expected {api_resource_name} to not exist. Did previous test cleanup?" ) logging.debug( f"http api resource. name: {api_resource_name}, data: {api_data}") k8s.create_custom_resource(api_ref, api_data) cr = k8s.wait_resource_consumed_by_controller(api_ref) assert cr is not None assert k8s.get_resource_exists(api_ref) api_id = cr['status']['apiID'] test_resource_values['API_ID'] = api_id yield api_ref, cr k8s.delete_custom_resource(api_ref)
def stage_resource(route_resource): random_suffix = (''.join( random.choice(string.ascii_lowercase) for _ in range(6))) stage_resource_name = test_resource_values[ 'STAGE_NAME'] + f'-{random_suffix}' test_resource_values['STAGE_NAME'] = stage_resource_name stage_ref, stage_data = helper.stage_ref_and_data( stage_resource_name=stage_resource_name, replacement_values=test_resource_values) if k8s.get_resource_exists(stage_ref): raise Exception( f"expected {stage_resource_name} to not exist. Did previous test cleanup?" ) logging.debug( f"apigatewayv2 stage resource. name: {stage_resource_name}, data: {stage_data}" ) k8s.create_custom_resource(stage_ref, stage_data) cr = k8s.wait_resource_consumed_by_controller(stage_ref) assert cr is not None assert k8s.get_resource_exists(stage_ref) yield stage_ref, cr k8s.delete_custom_resource(stage_ref)
def integration_resource(api_resource): random_suffix = (''.join( random.choice(string.ascii_lowercase) for _ in range(6))) integration_resource_name = test_resource_values[ 'INTEGRATION_NAME'] + f'-{random_suffix}' test_resource_values['INTEGRATION_NAME'] = integration_resource_name integration_ref, integration_data = helper.integration_ref_and_data( integration_resource_name=integration_resource_name, replacement_values=test_resource_values) if k8s.get_resource_exists(integration_ref): raise Exception( f"expected {integration_resource_name} to not exist. Did previous test cleanup?" ) logging.debug( f"apigatewayv2 integration resource. name: {integration_resource_name}, data: {integration_data}" ) k8s.create_custom_resource(integration_ref, integration_data) cr = k8s.wait_resource_consumed_by_controller(integration_ref) assert cr is not None assert k8s.get_resource_exists(integration_ref) integration_id = cr['status']['integrationID'] test_resource_values['INTEGRATION_ID'] = integration_id yield integration_ref, cr k8s.delete_custom_resource(integration_ref)
def single_variant_config(): config_resource_name = random_suffix_name("single-variant-config", 32) model_resource_name = config_resource_name + "-model" replacements = REPLACEMENT_VALUES.copy() replacements["CONFIG_NAME"] = config_resource_name replacements["MODEL_NAME"] = model_resource_name model_reference, model_spec, model_resource = create_sagemaker_resource( resource_plural=MODEL_RESOURCE_PLURAL, resource_name=model_resource_name, spec_file="xgboost_model", replacements=replacements, ) assert model_resource is not None config_reference, config_spec, config_resource = create_sagemaker_resource( resource_plural=CONFIG_RESOURCE_PLURAL, resource_name=config_resource_name, spec_file="endpoint_config_single_variant", replacements=replacements, ) assert config_resource is not None yield (config_reference, config_resource) k8s.delete_custom_resource(model_reference) # Delete the k8s resource if not already deleted by tests if k8s.get_resource_exists(config_reference): k8s.delete_custom_resource(config_reference)
def test_crud_authorizer(self, api_resource): api_ref, api_cr = api_resource api_id = api_cr['status']['apiID'] test_data = REPLACEMENT_VALUES.copy() authorizer_name = random_suffix_name("ack-test-authorizer", 25) test_data['AUTHORIZER_NAME'] = authorizer_name test_data['AUTHORIZER_TITLE'] = authorizer_name test_data['API_ID'] = api_id test_data['AUTHORIZER_URI'] = f'arn:aws:apigateway:{get_region()}:lambda:path/2015-03-31/functions/{get_bootstrap_resources().AuthorizerFunctionArn}/invocations' authorizer_ref, authorizer_data = helper.authorizer_ref_and_data(authorizer_resource_name=authorizer_name, replacement_values=test_data) logging.debug(f"http api authorizer resource. name: {authorizer_name}, data: {authorizer_data}") # test create k8s.create_custom_resource(authorizer_ref, authorizer_data) time.sleep(CREATE_WAIT_AFTER_SECONDS) assert k8s.wait_on_condition(authorizer_ref, "ACK.ResourceSynced", "True", wait_periods=10) cr = k8s.get_resource(authorizer_ref) assert cr is not None authorizer_id = cr['status']['authorizerID'] # Let's check that the HTTP Api integration appears in Amazon API Gateway apigw_validator.assert_authorizer_is_present(api_id=api_id, authorizer_id=authorizer_id) apigw_validator.assert_authorizer_name( api_id=api_id, authorizer_id=authorizer_id, expected_authorizer_name=authorizer_name ) # test update updated_authorizer_title = 'updated-' + authorizer_name test_data['AUTHORIZER_TITLE'] = updated_authorizer_title updated_authorizer_resource_data = load_apigatewayv2_resource( "authorizer", additional_replacements=test_data, ) logging.debug(f"updated http api authorizer resource: {updated_authorizer_resource_data}") # Update the k8s resource k8s.patch_custom_resource(authorizer_ref, updated_authorizer_resource_data) time.sleep(UPDATE_WAIT_AFTER_SECONDS) condition.assert_synced(authorizer_ref) # Let's check that the HTTP Api authorizer appears in Amazon API Gateway with updated title apigw_validator.assert_authorizer_name( api_id=api_id, authorizer_id=authorizer_id, expected_authorizer_name=updated_authorizer_title ) # test delete k8s.delete_custom_resource(authorizer_ref) time.sleep(DELETE_WAIT_AFTER_SECONDS) assert not k8s.get_resource_exists(authorizer_ref) # HTTP Api authorizer should no longer appear in Amazon API Gateway apigw_validator.assert_authorizer_is_deleted(api_id=api_id, authorizer_id=authorizer_id)
def test_crud_stage(self, api_resource): api_ref, api_cr = api_resource api_id = api_cr['status']['apiID'] test_data = REPLACEMENT_VALUES.copy() random_suffix = (''.join( random.choice(string.ascii_lowercase) for _ in range(6))) stage_name = "ack-test-stage-" + random_suffix test_data['STAGE_NAME'] = stage_name test_data['API_ID'] = api_id stage_ref, stage_data = helper.stage_ref_and_data( stage_resource_name=stage_name, replacement_values=test_data) logging.debug( f"http api stage resource. name: {stage_name}, data: {stage_data}") # test create k8s.create_custom_resource(stage_ref, stage_data) cr = k8s.wait_resource_consumed_by_controller(stage_ref) assert cr is not None assert k8s.get_resource_exists(stage_ref) # Let's check that the HTTP Api integration appears in Amazon API Gateway apigw_validator.assert_stage_is_present(api_id=api_id, stage_name=stage_name) stage_description = test_data['STAGE_DESCRIPTION'] apigw_validator.assert_stage_description( api_id=api_id, stage_name=stage_name, expected_description=stage_description) # test update updated_description = 'updated' + stage_description test_data['STAGE_DESCRIPTION'] = updated_description updated_stage_resource_data = load_apigatewayv2_resource( "stage", additional_replacements=test_data, ) logging.debug( f"updated http api stage resource: {updated_stage_resource_data}") # Update the k8s resource k8s.patch_custom_resource(stage_ref, updated_stage_resource_data) time.sleep(UPDATE_WAIT_AFTER_SECONDS) # Let's check that the HTTP Api stage appears in Amazon API Gateway with updated description apigw_validator.assert_stage_description( api_id=api_id, stage_name=stage_name, expected_description=updated_description) # test delete k8s.delete_custom_resource(stage_ref) time.sleep(DELETE_WAIT_AFTER_SECONDS) assert not k8s.get_resource_exists(stage_ref) # HTTP Api stage should no longer appear in Amazon API Gateway apigw_validator.assert_stage_is_deleted(api_id=api_id, stage_name=stage_name)
def test_create_delete_2az(self, rds_client): resource_name = "my-db-subnet-group" resource_desc = "my-db-subnet-group description" br_resources = get_bootstrap_resources() replacements = REPLACEMENT_VALUES.copy() replacements["DB_SUBNET_GROUP_NAME"] = resource_name replacements["DB_SUBNET_GROUP_DESC"] = resource_desc replacements["SUBNET_AZ1"] = br_resources.SubnetAZ1 replacements["SUBNET_AZ2"] = br_resources.SubnetAZ2 resource_data = load_rds_resource( "db_subnet_group_2az", additional_replacements=replacements, ) logging.debug(resource_data) # Create the k8s resource ref = k8s.CustomResourceReference( CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL, resource_name, namespace="default", ) k8s.create_custom_resource(ref, resource_data) cr = k8s.wait_resource_consumed_by_controller(ref) assert cr is not None assert k8s.get_resource_exists(ref) # Let's check that the DB subnet group appears in RDS aws_res = rds_client.describe_db_subnet_groups(DBSubnetGroupName=resource_name) assert aws_res is not None assert len(aws_res['DBSubnetGroups']) == 1 now = datetime.datetime.now() timeout = now + datetime.timedelta(seconds=CREATE_TIMEOUT_SECONDS) # TODO(jaypipes): Move this into generic AWS-side waiter while aws_res['DBSubnetGroups'][0]['SubnetGroupStatus'] != "Complete": if datetime.datetime.now() >= timeout: raise Exception("failed to find DB subnet group in Complete status before timeout") time.sleep(CREATE_INTERVAL_SLEEP_SECONDS) aws_res = rds_client.describe_db_subnet_groups(DBSubnetGroupName=resource_name) assert aws_res is not None assert len(aws_res['DBSubnetGroups']) == 1 # Delete the k8s resource on teardown of the module k8s.delete_custom_resource(ref) time.sleep(DELETE_WAIT_AFTER_SECONDS) # DB subnet group should no longer appear in RDS try: aws_res = rds_client.describe_db_subnet_groups(DBSubnetGroupName=resource_name) assert False except rds_client.exceptions.DBSubnetGroupNotFoundFault: pass
def test_crud_integration(self, api_resource): api_ref, api_cr = api_resource api_id = api_cr['status']['apiID'] test_data = REPLACEMENT_VALUES.copy() integration_name = random_suffix_name("ack-test-integration", 25) test_data['INTEGRATION_NAME'] = integration_name test_data['API_ID'] = api_id integration_ref, integration_data = helper.integration_ref_and_data(integration_resource_name=integration_name, replacement_values=test_data) logging.debug(f"http api integration resource. name: {integration_name}, data: {integration_data}") # test create k8s.create_custom_resource(integration_ref, integration_data) time.sleep(CREATE_WAIT_AFTER_SECONDS) assert k8s.wait_on_condition(integration_ref, "ACK.ResourceSynced", "True", wait_periods=10) cr = k8s.get_resource(integration_ref) assert cr is not None integration_id = cr['status']['integrationID'] # Let's check that the HTTP Api integration appears in Amazon API Gateway apigw_validator.assert_integration_is_present(api_id=api_id, integration_id=integration_id) apigw_validator.assert_integration_uri( api_id=api_id, integration_id=integration_id, expected_uri=test_data['INTEGRATION_URI'] ) # test update updated_uri = 'https://httpbin.org/post' test_data['INTEGRATION_URI'] = updated_uri updated_integration_resource_data = load_apigatewayv2_resource( "integration", additional_replacements=test_data, ) logging.debug(f"updated http api integration resource: {updated_integration_resource_data}") # Update the k8s resource k8s.patch_custom_resource(integration_ref, updated_integration_resource_data) time.sleep(UPDATE_WAIT_AFTER_SECONDS) condition.assert_synced(integration_ref) # Let's check that the HTTP Api integration appears in Amazon API Gateway with updated uri apigw_validator.assert_integration_uri( api_id=api_id, integration_id=integration_id, expected_uri=updated_uri ) # test delete k8s.delete_custom_resource(integration_ref) time.sleep(DELETE_WAIT_AFTER_SECONDS) assert not k8s.get_resource_exists(integration_ref) # HTTP Api integration should no longer appear in Amazon API Gateway apigw_validator.assert_integration_is_deleted(api_id=api_id, integration_id=integration_id)
def test_crud_stage(self, api_resource): api_ref, api_cr = api_resource api_id = api_cr['status']['apiID'] test_data = REPLACEMENT_VALUES.copy() stage_name = random_suffix_name("ack-test-stage", 25) test_data['STAGE_NAME'] = stage_name test_data['API_ID'] = api_id stage_ref, stage_data = helper.stage_ref_and_data(stage_resource_name=stage_name, replacement_values=test_data) logging.debug(f"http api stage resource. name: {stage_name}, data: {stage_data}") # test create k8s.create_custom_resource(stage_ref, stage_data) time.sleep(CREATE_WAIT_AFTER_SECONDS) assert k8s.wait_on_condition(stage_ref, "ACK.ResourceSynced", "True", wait_periods=10) cr = k8s.get_resource(stage_ref) assert cr is not None # Let's check that the HTTP Api integration appears in Amazon API Gateway apigw_validator.assert_stage_is_present(api_id=api_id, stage_name=stage_name) stage_description = test_data['STAGE_DESCRIPTION'] apigw_validator.assert_stage_description( api_id=api_id, stage_name=stage_name, expected_description=stage_description ) # test update updated_description = 'updated' + stage_description test_data['STAGE_DESCRIPTION'] = updated_description updated_stage_resource_data = load_apigatewayv2_resource( "stage", additional_replacements=test_data, ) logging.debug(f"updated http api stage resource: {updated_stage_resource_data}") # Update the k8s resource k8s.patch_custom_resource(stage_ref, updated_stage_resource_data) time.sleep(UPDATE_WAIT_AFTER_SECONDS) condition.assert_synced(stage_ref) # Let's check that the HTTP Api stage appears in Amazon API Gateway with updated description apigw_validator.assert_stage_description( api_id=api_id, stage_name=stage_name, expected_description=updated_description ) # test delete k8s.delete_custom_resource(stage_ref) time.sleep(DELETE_WAIT_AFTER_SECONDS) assert not k8s.get_resource_exists(stage_ref) # HTTP Api stage should no longer appear in Amazon API Gateway apigw_validator.assert_stage_is_deleted(api_id=api_id, stage_name=stage_name)
def rg_cmd_update(rg_cmd_update_input, make_replication_group, rg_deletion_waiter): input_dict = rg_cmd_update_input (reference, resource) = make_replication_group("replicationgroup_cmd_update", input_dict, input_dict["RG_ID"]) yield (reference, resource) # teardown k8s.delete_custom_resource(reference) sleep(DEFAULT_WAIT_SECS) rg_deletion_waiter.wait(ReplicationGroupId=input_dict["RG_ID"])
def rg_auth_token(make_rg_name, make_replication_group, rg_deletion_waiter, secrets): input_dict = { "RG_ID": make_rg_name("rg-auth-token"), "NAME": secrets['NAME1'], "KEY": secrets['KEY1'] } (reference, resource) = make_replication_group("replicationgroup_authtoken", input_dict, input_dict["RG_ID"]) yield (reference, resource) k8s.delete_custom_resource(reference) sleep(DEFAULT_WAIT_SECS) rg_deletion_waiter.wait(ReplicationGroupId=input_dict["RG_ID"]) # throws exception if wait fails
def test_crud_httpapi_using_import(self): test_data = REPLACEMENT_VALUES.copy() api_name = random_suffix_name("ack-test-importapi", 25) test_data['IMPORT_API_NAME'] = api_name test_data['IMPORT_API_TITLE'] = api_name api_ref, api_data = helper.import_api_ref_and_data(api_resource_name=api_name, replacement_values=test_data) logging.debug(f"imported http api resource. name: {api_name}, data: {api_data}") # test create k8s.create_custom_resource(api_ref, api_data) time.sleep(CREATE_API_WAIT_AFTER_SECONDS) assert k8s.wait_on_condition(api_ref, "ACK.ResourceSynced", "True", wait_periods=10) cr = k8s.get_resource(api_ref) assert cr is not None api_id = cr['status']['apiID'] # Let's check that the imported HTTP Api appears in Amazon API Gateway apigw_validator.assert_api_is_present(api_id=api_id) apigw_validator.assert_api_name( api_id=api_id, expected_api_name=api_name ) # test update updated_api_title = 'updated-' + api_name test_data['IMPORT_API_TITLE'] = updated_api_title updated_api_resource_data = load_apigatewayv2_resource( "import_api", additional_replacements=test_data, ) logging.debug(f"updated import http api resource: {updated_api_resource_data}") # Update the k8s resource k8s.patch_custom_resource(api_ref, updated_api_resource_data) time.sleep(UPDATE_WAIT_AFTER_SECONDS) condition.assert_synced(api_ref) # Let's check that the HTTP Api appears in Amazon API Gateway with updated title apigw_validator.assert_api_name( api_id=api_id, expected_api_name=updated_api_title ) # test delete k8s.delete_custom_resource(api_ref) time.sleep(DELETE_WAIT_AFTER_SECONDS) assert not k8s.get_resource_exists(api_ref) # HTTP Api should no longer appear in Amazon API Gateway apigw_validator.assert_api_is_deleted(api_id=api_id)
def rg_cmd_fromsnapshot(bootstrap_resources, make_rg_name, make_replication_group, rg_deletion_waiter): input_dict = { "RG_ID": make_rg_name("rg-cmd-fromsnapshot"), "SNAPSHOT_NAME": bootstrap_resources.SnapshotName } (reference, resource) = make_replication_group("replicationgroup_cmd_fromsnapshot", input_dict, input_dict["RG_ID"]) yield (reference, resource) # teardown k8s.delete_custom_resource(reference) sleep(DEFAULT_WAIT_SECS) rg_deletion_waiter.wait(ReplicationGroupId=input_dict["RG_ID"])
def test_crud_httpapi_using_import(self): test_data = REPLACEMENT_VALUES.copy() random_suffix = (''.join( random.choice(string.ascii_lowercase) for _ in range(6))) api_name = "ack-test-importapi-" + random_suffix test_data['API_NAME'] = api_name test_data['API_TITLE'] = api_name api_ref, api_data = helper.import_api_ref_and_data( api_resource_name=api_name, replacement_values=test_data) logging.debug( f"imported http api resource. name: {api_name}, data: {api_data}") # test create k8s.create_custom_resource(api_ref, api_data) cr = k8s.wait_resource_consumed_by_controller(api_ref) assert cr is not None assert k8s.get_resource_exists(api_ref) api_id = cr['status']['apiID'] # Let's check that the imported HTTP Api appears in Amazon API Gateway apigw_validator.assert_api_is_present(api_id=api_id) apigw_validator.assert_api_name(api_id=api_id, expected_api_name=api_name) # test update updated_api_title = 'updated-' + api_name test_data['API_TITLE'] = updated_api_title updated_api_resource_data = load_apigatewayv2_resource( "import_api", additional_replacements=test_data, ) logging.debug( f"updated import http api resource: {updated_api_resource_data}") # Update the k8s resource k8s.patch_custom_resource(api_ref, updated_api_resource_data) time.sleep(UPDATE_WAIT_AFTER_SECONDS) # Let's check that the HTTP Api appears in Amazon API Gateway with updated title apigw_validator.assert_api_name(api_id=api_id, expected_api_name=updated_api_title) # test delete k8s.delete_custom_resource(api_ref) time.sleep(DELETE_WAIT_AFTER_SECONDS) assert not k8s.get_resource_exists(api_ref) # HTTP Api should no longer appear in Amazon API Gateway apigw_validator.assert_api_is_deleted(api_id=api_id)
def user_password(user_password_input, elasticache_client): # inject parameters into yaml; create User in cluster user = load_elasticache_resource("user_password", additional_replacements=user_password_input) reference = k8s.CustomResourceReference( CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL, user_password_input["USER_ID"], namespace="default") _ = k8s.create_custom_resource(reference, user) resource = k8s.wait_resource_consumed_by_controller(reference) assert resource is not None yield (reference, resource) # teardown: delete in k8s, assert user does not exist in AWS k8s.delete_custom_resource(reference) sleep(DEFAULT_WAIT_SECS) assert_user_deletion(user_password_input['USER_ID'])
def test_create_delete_simple(self, rds_client): resource_name = "my-db-security-group" resource_desc = "my-db-security-group description" br_resources = get_bootstrap_resources() replacements = REPLACEMENT_VALUES.copy() replacements["DB_SECURITY_GROUP_NAME"] = resource_name replacements["DB_SECURITY_GROUP_DESC"] = resource_desc resource_data = load_rds_resource( "db_security_group_simple", additional_replacements=replacements, ) logging.debug(resource_data) # Create the k8s resource ref = k8s.CustomResourceReference( CRD_GROUP, CRD_VERSION, RESOURCE_PLURAL, resource_name, namespace="default", ) k8s.create_custom_resource(ref, resource_data) cr = k8s.wait_resource_consumed_by_controller(ref) assert cr is not None assert k8s.get_resource_exists(ref) # Let's check that the DB security group appears in RDS aws_res = rds_client.describe_db_security_groups( DBSecurityGroupName=resource_name) assert aws_res is not None assert len(aws_res['DBSecurityGroups']) == 1 # Delete the k8s resource on teardown of the module k8s.delete_custom_resource(ref) time.sleep(DELETE_WAIT_AFTER_SECONDS) # DB security group should no longer appear in RDS try: aws_res = rds_client.describe_db_security_groups( DBSecurityGroupName=resource_name) assert False except rds_client.exceptions.DBSecurityGroupNotFoundFault: pass
def test_completed(self, kmeans_processing_job): (reference, resource) = kmeans_processing_job assert k8s.get_resource_exists(reference) processing_job_name = resource["spec"].get("processingJobName", None) assert processing_job_name is not None processing_job_desc = get_sagemaker_processing_job(processing_job_name) processing_job_arn = processing_job_desc["ProcessingJobArn"] assert k8s.get_resource_arn(resource) == processing_job_arn assert processing_job_desc["ProcessingJobStatus"] == cfg.JOB_STATUS_INPROGRESS assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False") self._assert_processing_status_in_sync( processing_job_name, reference, cfg.JOB_STATUS_COMPLETED ) assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True") resource_tags = resource["spec"].get("tags", None) assert_tags_in_sync(processing_job_arn, resource_tags) # Check that you can delete a completed resource from k8s _, deleted = k8s.delete_custom_resource(reference, cfg.JOB_DELETE_WAIT_PERIODS, cfg.JOB_DELETE_WAIT_LENGTH) assert deleted is True
def test_completed(self, xgboost_training_job): (reference, resource) = xgboost_training_job assert k8s.get_resource_exists(reference) training_job_name = resource["spec"].get("trainingJobName", None) assert training_job_name is not None training_job_desc = get_sagemaker_training_job(training_job_name) training_job_arn = training_job_desc["TrainingJobArn"] assert k8s.get_resource_arn(resource) == training_job_arn assert training_job_desc[ "TrainingJobStatus"] == cfg.JOB_STATUS_INPROGRESS assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False") assert_training_status_in_sync(training_job_name, reference, cfg.JOB_STATUS_COMPLETED) assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True") # model artifact URL is populated resource = k8s.get_resource(reference) resource["status"]["modelArtifacts"]["s3ModelArtifacts"] is not None resource_tags = resource["spec"].get("tags", None) assert_tags_in_sync(training_job_arn, resource_tags) # Check that you can delete a completed resource from k8s _, deleted = k8s.delete_custom_resource(reference, cfg.JOB_DELETE_WAIT_PERIODS, cfg.JOB_DELETE_WAIT_LENGTH) assert deleted is True
def test_stopped(self, xgboost_training_job): (reference, resource) = xgboost_training_job assert k8s.get_resource_exists(reference) training_job_name = resource["spec"].get("trainingJobName", None) assert training_job_name is not None training_job_desc = get_sagemaker_training_job(training_job_name) assert k8s.get_resource_arn( resource) == training_job_desc["TrainingJobArn"] assert training_job_desc[ "TrainingJobStatus"] == cfg.JOB_STATUS_INPROGRESS assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False") assert_training_status_in_sync(training_job_name, reference, cfg.JOB_STATUS_INPROGRESS) # Delete the k8s resource. _, deleted = k8s.delete_custom_resource(reference, cfg.JOB_DELETE_WAIT_PERIODS, cfg.JOB_DELETE_WAIT_LENGTH) assert deleted is True training_job_desc = get_sagemaker_training_job(training_job_name) assert training_job_desc[ "TrainingJobStatus"] in cfg.LIST_JOB_STATUS_STOPPED
def xgboost_training_job(): resource_name = random_suffix_name("xgboost-trainingjob", 32) replacements = REPLACEMENT_VALUES.copy() replacements["TRAINING_JOB_NAME"] = resource_name reference, _, resource = create_sagemaker_resource( resource_plural=RESOURCE_PLURAL, resource_name=resource_name, spec_file="xgboost_trainingjob", replacements=replacements, ) assert resource is not None if k8s.get_resource_arn(resource) is None: logging.error( f"ARN for this resource is None, resource status is: {resource['status']}" ) assert k8s.get_resource_arn(resource) is not None yield (reference, resource) if k8s.get_resource_exists(reference): _, deleted = k8s.delete_custom_resource(reference, cfg.JOB_DELETE_WAIT_PERIODS, cfg.JOB_DELETE_WAIT_LENGTH) assert deleted
def single_container_model(name_suffix): model_resource_name = name_suffix + "-model" replacements = REPLACEMENT_VALUES.copy() replacements["MODEL_NAME"] = model_resource_name model_reference, model_spec, model_resource = create_sagemaker_resource( resource_plural=MODEL_RESOURCE_PLURAL, resource_name=model_resource_name, spec_file="xgboost_model", replacements=replacements, ) assert model_resource is not None yield (model_reference, model_resource) k8s.delete_custom_resource(model_reference)
def test_smoke( self, sagemaker_client, xgboost_churn_model_explainability_job_definition ): (reference, resource) = xgboost_churn_model_explainability_job_definition assert k8s.get_resource_exists(reference) job_definition_name = resource["spec"].get("jobDefinitionName") job_definition_desc = get_sagemaker_model_explainability_job_definition( sagemaker_client, job_definition_name ) job_definition_arn = job_definition_desc["JobDefinitionArn"] assert k8s.get_resource_arn(resource) == job_definition_arn # random sleep before we check for tags to reduce test flakyness time.sleep(cfg.TAG_DELAY_SLEEP) resource_tags = resource["spec"].get("tags", None) assert_tags_in_sync(job_definition_arn, resource_tags) # Delete the k8s resource. _, deleted = k8s.delete_custom_resource(reference, 3, 10) assert deleted assert ( get_sagemaker_model_explainability_job_definition( sagemaker_client, job_definition_name ) is None )
def xgboost_endpoint(name_suffix, single_variant_config): endpoint_resource_name = name_suffix (_, config_resource) = single_variant_config config_resource_name = config_resource["spec"].get("endpointConfigName", None) replacements = REPLACEMENT_VALUES.copy() replacements["ENDPOINT_NAME"] = endpoint_resource_name replacements["ENDPOINT_CONFIG_NAME"] = config_resource_name reference, spec, resource = create_sagemaker_resource( resource_plural=cfg.ENDPOINT_RESOURCE_PLURAL, resource_name=endpoint_resource_name, spec_file="endpoint_base", replacements=replacements, ) assert resource is not None yield (reference, resource, spec) # Delete the k8s resource if not already deleted by tests if k8s.get_resource_exists(reference): # longer wait incase endpoint is in creating/updating status _, deleted = k8s.delete_custom_resource(reference, 40, cfg.DELETE_WAIT_LENGTH) assert deleted
def multi_variant_config(name_suffix, single_container_model): config_resource_name = name_suffix + "-multi-variant-config" (_, model_resource) = single_container_model model_resource_name = model_resource["spec"].get("modelName", None) replacements = REPLACEMENT_VALUES.copy() replacements["ENDPOINT_CONFIG_NAME"] = config_resource_name replacements["MODEL_NAME"] = model_resource_name config_reference, config_spec, config_resource = create_sagemaker_resource( resource_plural=cfg.ENDPOINT_CONFIG_RESOURCE_PLURAL, resource_name=config_resource_name, spec_file="endpoint_config_multi_variant", replacements=replacements, ) assert config_resource is not None if k8s.get_resource_arn(config_resource) is None: logging.error( f"ARN for this resource is None, resource status is: {config_resource['status']}" ) assert k8s.get_resource_arn(config_resource) is not None yield (config_reference, config_resource) _, deleted = k8s.delete_custom_resource(config_reference, 3, 10) assert deleted
def xgboost_model_package_group(): resource_name = random_suffix_name("xgboost-model-package-group", 38) replacements = REPLACEMENT_VALUES.copy() replacements["MODEL_PACKAGE_GROUP_NAME"] = resource_name ( model_package_group_reference, model_package_group_spec, model_package_group_resource, ) = create_sagemaker_resource( resource_plural=cfg.MODEL_PACKAGE_GROUP_RESOURCE_PLURAL, resource_name=resource_name, spec_file="xgboost_model_package_group", replacements=replacements, ) assert model_package_group_resource is not None if k8s.get_resource_arn(model_package_group_resource) is None: logging.error( f"ARN for this resource is None, resource status is: {model_package_group_resource['status']}" ) assert k8s.get_resource_arn(model_package_group_resource) is not None yield (model_package_group_reference, model_package_group_resource) # Delete the k8s resource if not already deleted by tests if k8s.get_resource_exists(model_package_group_reference): _, deleted = k8s.delete_custom_resource(model_package_group_reference, DELETE_WAIT_PERIOD, DELETE_WAIT_LENGTH) assert deleted
def test_stopped(self, xgboost_hpojob): (reference, resource) = xgboost_hpojob assert k8s.get_resource_exists(reference) hpo_job_name = resource["spec"].get("hyperParameterTuningJobName", None) assert hpo_job_name is not None hpo_sm_desc = get_sagemaker_hpo_job(hpo_job_name) assert (k8s.get_resource_arn(resource) == hpo_sm_desc["HyperParameterTuningJobArn"]) assert hpo_sm_desc[ "HyperParameterTuningJobStatus"] == cfg.JOB_STATUS_INPROGRESS assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False") self._assert_hpo_status_in_sync(hpo_job_name, reference, cfg.JOB_STATUS_INPROGRESS) # Delete the k8s resource. _, deleted = k8s.delete_custom_resource(reference, cfg.JOB_DELETE_WAIT_PERIODS, cfg.JOB_DELETE_WAIT_LENGTH) assert deleted is True hpo_sm_desc = get_sagemaker_hpo_job(hpo_job_name) assert (hpo_sm_desc["HyperParameterTuningJobStatus"] in cfg.LIST_JOB_STATUS_STOPPED)
def test_create_feature_group(self, feature_group): """Tests that a feature group can be created and deleted using the Feature Group Controller. """ (reference, resource) = feature_group assert k8s.get_resource_exists(reference) feature_group_name = resource["spec"].get("featureGroupName", None) assert feature_group_name is not None feature_group_sm_desc = get_sagemaker_feature_group(feature_group_name) feature_group_arn = feature_group_sm_desc["FeatureGroupArn"] assert k8s.get_resource_arn(resource) == feature_group_arn assert feature_group_sm_desc[ "FeatureGroupStatus"] == FEATURE_GROUP_STATUS_CREATING assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False") self._assert_feature_group_status_in_sync( feature_group_name, reference, FEATURE_GROUP_STATUS_CREATED) assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True") resource_tags = resource["spec"].get("tags", None) assert_tags_in_sync(feature_group_arn, resource_tags) # Delete the k8s resource. _, deleted = k8s.delete_custom_resource(reference, WAIT_PERIOD_COUNT, WAIT_PERIOD_LENGTH) assert deleted assert get_sagemaker_feature_group(feature_group_name) is None
def test_completed(self, xgboost_hpojob): (reference, resource) = xgboost_hpojob assert k8s.get_resource_exists(reference) hpo_job_name = resource["spec"].get("hyperParameterTuningJobName", None) assert hpo_job_name is not None hpo_sm_desc = get_sagemaker_hpo_job(hpo_job_name) hpo_arn = hpo_sm_desc["HyperParameterTuningJobArn"] assert k8s.get_resource_arn(resource) == hpo_arn assert hpo_sm_desc[ "HyperParameterTuningJobStatus"] == cfg.JOB_STATUS_INPROGRESS assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "False") self._assert_hpo_status_in_sync(hpo_job_name, reference, cfg.JOB_STATUS_COMPLETED) assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True") resource_tags = resource["spec"].get("tags", None) assert_tags_in_sync(hpo_arn, resource_tags) # Check that you can delete a completed resource from k8s _, deleted = k8s.delete_custom_resource(reference, cfg.JOB_DELETE_WAIT_PERIODS, cfg.JOB_DELETE_WAIT_LENGTH) assert deleted is True
def xgboost_churn_data_quality_job_definition(xgboost_churn_endpoint): endpoint_spec = xgboost_churn_endpoint endpoint_name = endpoint_spec["spec"].get("endpointName") resource_name = random_suffix_name("data-quality-job-definition", 32) replacements = REPLACEMENT_VALUES.copy() replacements["JOB_DEFINITION_NAME"] = resource_name replacements["ENDPOINT_NAME"] = endpoint_name job_definition_reference, _, resource = create_sagemaker_resource( resource_plural=cfg.DATA_QUALITY_JOB_DEFINITION_RESOURCE_PLURAL, resource_name=resource_name, spec_file="data_quality_job_definition_xgboost_churn", replacements=replacements, ) assert resource is not None job_definition_name = resource["spec"].get("jobDefinitionName") yield (job_definition_reference, resource) if k8s.get_resource_exists(job_definition_reference): _, deleted = k8s.delete_custom_resource(job_definition_reference, 3, 10) assert deleted