def mock_cognito_create_user_add_user_to_group_fails(admin_user,
create_user_arguments):
_keep_it_real()
client = boto3.real_client("cognito-idp")
group_name = admin_user["group"]["value"]
stubber = Stubber(client)
# Add responses
stub_response_cognito_admin_create_user(stubber, admin_user,
create_user_arguments)
stub_response_cognito_admin_set_user_mfa_preference(
stubber, admin_user["email"])
stub_response_cognito_admin_set_user_settings(stubber, admin_user["email"])
# stub_response_cognito_admin_add_user_to_group(
# stubber, admin_user["email"], group_name
# )
stubber.add_client_error(
"admin_add_user_to_group",
expected_params={
"UserPoolId": MOCK_COGNITO_USER_POOL_ID,
"Username": admin_user["email"],
"GroupName": group_name,
},
)
stub_response_cognito_admin_disable_user(stubber, admin_user["email"])
stubber.activate()
# override boto.client to return the mock client
boto3.client = lambda service, region_name=None: client
return stubber
def test_putRecord_err(self, mock_convert):
kinesis = OutputManager()
stubber = Stubber(kinesis.KINESIS_CLIENT)
mock_data = MagicMock()
mock_data.test = 'data'
record = {'type': 'work', 'method': 'update', 'data': mock_data}
expected_params = {
'Data': 'testing',
'StreamName': 'tester',
'PartitionKey': 'testUUID'
}
expected_params = {
'Data': 'testing',
'StreamName': 'tester',
'PartitionKey': 'testUUID'
}
stubber.add_client_error('put_record', expected_params=expected_params)
stubber.activate()
try:
kinesis.putRecord(record, 'testStream', 'testUUID')
except KinesisError:
pass
self.assertRaises(KinesisError)
def _boto3_stubber(service, mocked_requests):
client = boto3.client(service)
stubber = Stubber(client)
# Save a ref to the stubber so that we can deactivate it at the end of the test.
created_stubbers.append(stubber)
# Attach mocked requests to the Stubber and activate it.
if not isinstance(mocked_requests, list):
mocked_requests = [mocked_requests]
for mocked_request in mocked_requests:
if mocked_request.generate_error:
stubber.add_client_error(
mocked_request.method,
service_message=mocked_request.response,
expected_params=mocked_request.expected_params,
service_error_code=mocked_request.error_code,
)
else:
stubber.add_response(
mocked_request.method,
mocked_request.response,
expected_params=mocked_request.expected_params)
stubber.activate()
# Add stubber to the collection of mocked clients. This allows to mock multiple clients.
# Mocking twice the same client will replace the previous one.
mocked_clients[service] = client
return client
def test_persistent_graph_no_such_key(self, mocker: MockerFixture) -> None:
"""Test persistent_graph NoSuchKey."""
mocker.patch.object(
CfnginContext,
"persistent_graph_location",
{
"Bucket": "test-bucket",
"Key": "something.json"
},
)
mocker.patch.object(CfnginContext, "s3_bucket_verified", True)
obj = CfnginContext()
stubber = Stubber(obj.s3_client)
stubber.add_client_error("get_object", "NoSuchKey")
stubber.add_response(
"put_object",
{},
{
"Body": "{}".encode(),
"ServerSideEncryption": "AES256",
"ACL": "bucket-owner-full-control",
"ContentType": "application/json",
**obj.persistent_graph_location,
},
)
with stubber:
assert isinstance(obj.persistent_graph, Graph)
assert obj.persistent_graph.to_dict() == {}
def test_unlock_persistent_graph_no_such_key(
self, mocker: MockerFixture) -> None:
"""Test unlock_persistent_graph empty graph NoSuchKey."""
mocker.patch.object(
CfnginContext,
"persistent_graph_location",
{
"Bucket": "test-bucket",
"Key": "something.json"
},
)
mocker.patch.object(CfnginContext, "persistent_graph_locked", True)
mocker.patch.object(CfnginContext, "persistent_graph_lock_code", "123")
obj = CfnginContext()
obj.persistent_graph = Graph.from_dict({}, context=obj)
stubber = Stubber(obj.s3_client)
stubber.add_response(
"get_object",
{"Body": "{}".encode()},
{
"ResponseContentType": "application/json",
**obj.persistent_graph_location,
},
)
stubber.add_client_error("delete_object_tagging", "NoSuchKey")
with stubber:
assert obj.unlock_persistent_graph("123")
def test_persistent_graph_no_object(self):
"""Create object if one does not exist and return empty Graph."""
context = Context(config=self.persist_graph_config)
context._s3_bucket_verified = True
stubber = Stubber(context.s3_conn)
expected_get_params = {'ResponseContentType': 'application/json'}
expected_get_params.update(context.persistent_graph_location)
expected_put_params = {
'Body': '{}',
'ServerSideEncryption': 'AES256',
'ACL': 'bucket-owner-full-control',
'ContentType': 'application/json'
}
expected_put_params.update(context.persistent_graph_location)
stubber.add_client_error('get_object',
'NoSuchKey',
expected_params=expected_get_params)
stubber.add_response('put_object', {}, expected_put_params)
with stubber:
self.assertIsNone(context._persistent_graph)
self.assertIsInstance(context.persistent_graph, Graph)
self.assertIsInstance(context._persistent_graph, Graph)
self.assertEqual({}, context.persistent_graph.to_dict())
stubber.assert_no_pending_responses()
def mock_s3_get_object(bucket_name, granted_prefixes, key, success_response):
_keep_it_real()
client = boto3.real_client("s3")
stubber = Stubber(client)
if any([key.startswith(prefix) for prefix in granted_prefixes]):
stubber.add_response(
"get_object",
success_response,
{
"Bucket": bucket_name,
"Key": key
},
)
else:
stubber.add_client_error("get_object",
expected_params={
"Bucket": bucket_name,
"Key": key
})
# replace the get_presigned_url so it runs without AWS creds
client.generate_presigned_url = lambda op, Params, ExpiresIn, HttpMethod: fake_url(
Params["Bucket"], Params["Key"])
stubber.activate()
# override boto.client to return the mock client
boto3.client = lambda service, region_name=None, config=None: client
return stubber
def test_get_snsname_arn_auth_exception_handling(self, aws_res_mock):
"""AuthorizationError is intercepted and re-raised as AssertionError"""
# local imports of code-under-test ensure moto has mocks
# registered before any possible calls out to AWS
from awstools.awstools import get_snsname_arn
# create a mock SNS client that returns what we tell it to
client = boto3.client('sns')
stub = Stubber(client)
stub.add_client_error('create_topic', service_error_code='AuthorizationError')
stub.activate()
# since firesim manager code doesn't take clients as method parameters
# now we mock boto3.client to return our stubbed client
with patch.object(boto3._get_default_session(), 'client', return_value=client) as mock_session:
topic_arn = get_snsname_arn()
stub.assert_no_pending_responses()
assert topic_arn == None
# TODO we could mock rootLogger.critical to capture it's calls and args and validate that we're seeing the correct "nice" message
# make sure get_snsname_arn() actually called out to get a sns
# client, otherwise we aren't testing what we think we are
mock_session.assert_called_once_with('sns')
aws_res_mock.assert_called_once()
def test_get_security_group_details_client_error():
ec2_client = botocore.session.get_session().create_client('ec2')
stubber = Stubber(ec2_client)
stubber.add_client_error('describe_security_groups')
with pytest.raises(ClientError) as e:
service_response = ec2_client.describe_security_groups(
GroupIds=['sg-abc1234'])
def mock_cognito_create_user_set_user_settings_fails(admin_user, create_user_arguments):
_keep_it_real()
client = boto3.real_client("cognito-idp")
group_name = admin_user["group"]["value"]
stubber = Stubber(client)
# Add responses
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_create_user(stubber, admin_user, create_user_arguments)
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_set_user_mfa_preference(stubber, admin_user["email"])
stub_response_cognito_list_user_pools(stubber)
# stub_response_cognito_admin_set_user_settings(stubber, admin_user["email"])
stubber.add_client_error(
"admin_set_user_settings",
expected_params={
"MFAOptions": [{"DeliveryMedium": "SMS", "AttributeName": "phone_number"}],
"UserPoolId": MOCK_COGNITO_USER_POOL_ID,
"Username": admin_user["email"],
},
)
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_add_user_to_group(
stubber, admin_user["email"], group_name
)
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_disable_user(stubber, admin_user["email"])
stubber.activate()
# override boto.client to return the mock client
boto3.client = lambda service, region_name=None: client
return stubber
def test_ensure_cfn_bucket_does_not_exist_us_west(self):
"""Test ensure cfn bucket does not exist us west."""
session = get_session("us-west-1")
provider = Provider(session)
action = BaseAction(
context=mock_context("mynamespace"),
provider_builder=MockProviderBuilder(provider, region="us-west-1"),
)
stubber = Stubber(action.s3_conn)
stubber.add_client_error(
"head_bucket",
service_error_code="NoSuchBucket",
service_message="Not Found",
http_status_code=404,
)
stubber.add_response(
"create_bucket",
service_response={},
expected_params={
"Bucket": ANY,
"CreateBucketConfiguration": {
"LocationConstraint": "us-west-1"
},
},
)
with stubber:
action.ensure_cfn_bucket()
def test_not_has_roles(self, sts_stubber: Stubber):
sts_stubber.add_client_error('assume_role')
sts_stubber.add_client_error('assume_role')
with pytest.raises(remediate.HttpInvalidException):
remediate.require_remediation_roles(
'*****@*****.**', 'arn:aws:iam::111:role/cross-account',
'arn:aws:iam::111:role/remediation-role')
def test_put_metric_catches_error(mock_session):
client = boto3.client("cloudwatch")
stubber = Stubber(client)
stubber.add_client_error("put_metric_data", "InternalServiceError")
stubber.activate()
mock_session.client.return_value = client
publisher = MetricPublisher(mock_session, NAMESPACE)
dimensions = {
"DimensionKeyActionType": Action.CREATE.name,
"DimensionKeyResourceType": RESOURCE_TYPE,
}
with patch("cloudformation_cli_python_lib.metrics.LOG",
auto_spec=True) as mock_logger:
publisher.publish_metric(
MetricTypes.HandlerInvocationCount,
dimensions,
StandardUnit.Count,
1.0,
datetime.now(),
)
stubber.deactivate()
expected_calls = [
call.error(
"An error occurred while publishing metrics: %s",
"An error occurred (InternalServiceError) when calling the "
"PutMetricData operation: ",
)
]
assert mock_logger.mock_calls == expected_calls
def _boto3_stubber(service, mocked_requests):
if "AWS_DEFAULT_REGION" not in os.environ:
# We need to provide a region to boto3 to avoid no region exception.
# Which region to provide is arbitrary.
os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
client = boto3.client(service)
stubber = Stubber(client)
# Save a ref to the stubber so that we can deactivate it at the end of the test.
created_stubbers.append(stubber)
# Attach mocked requests to the Stubber and activate it.
if not isinstance(mocked_requests, list):
mocked_requests = [mocked_requests]
for mocked_request in mocked_requests:
if mocked_request.generate_error:
stubber.add_client_error(
mocked_request.method,
service_message=mocked_request.response,
expected_params=mocked_request.expected_params,
service_error_code=mocked_request.error_code,
)
else:
stubber.add_response(
mocked_request.method, mocked_request.response, expected_params=mocked_request.expected_params
)
stubber.activate()
# Add stubber to the collection of mocked clients. This allows to mock multiple clients.
# Mocking twice the same client will replace the previous one.
mocked_clients[service] = client
return client
def mock_create_user_failure(admin_user, admin_get_user, create_user_arguments):
_keep_it_real()
client = boto3.real_client("cognito-idp")
stubber = Stubber(client)
# Add responses
# get user
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_get_user(stubber, admin_user["email"], admin_get_user)
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_list_groups_for_user(stubber, admin_user["email"])
# delete user
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_delete_user(stubber, admin_user["email"])
# create user
stub_response_cognito_list_user_pools(stubber)
stubber.add_client_error(
"admin_create_user", expected_params=create_user_arguments,
)
stubber.activate()
# override boto.client to return the mock client
boto3.client = lambda service, region_name=None: client
return stubber
def test_lock_persistent_graph_no_object(self):
"""Error raised when when there is no object to lock."""
code = "0000"
context = Context(config=self.persist_graph_config)
context._s3_bucket_verified = True
context._persistent_graph = Graph()
stubber = Stubber(context.s3_conn)
expected_params = {
"Tagging": {
"TagSet":
gen_tagset({context._persistent_graph_lock_tag: code})
}
}
expected_params.update(context.persistent_graph_location)
stubber.add_client_error(
"get_object_tagging",
"NoSuchKey",
expected_params=context.persistent_graph_location,
)
stubber.add_client_error("put_object_tagging",
"NoSuchKey",
expected_params=expected_params)
with stubber:
with self.assertRaises(PersistentGraphCannotLock):
context.lock_persistent_graph(code)
stubber.assert_no_pending_responses()
def test_s3_bucket_does_not_exist_us_west(self):
"""Create S3 bucket with loc constraints when it does not exist."""
region = "us-west-1"
context = Context(config=self.config, region=region)
stubber = Stubber(context.s3_conn)
stubber.add_client_error(
"head_bucket",
service_error_code="NoSuchBucket",
service_message="Not Found",
http_status_code=404,
)
stubber.add_response(
"create_bucket",
service_response={},
expected_params={
"Bucket": ANY,
"CreateBucketConfiguration": {
"LocationConstraint": region
},
},
)
with stubber:
self.assertIsNone(context._s3_bucket_verified)
self.assertTrue(context.s3_bucket_verified)
self.assertTrue(context._s3_bucket_verified)
stubber.assert_no_pending_responses()
def test_ensure_cfn_bucket_doesnt_exist_us_west(self):
session = get_session("us-west-1")
provider = Provider(session)
action = BaseAction(
context=mock_context("mynamespace"),
provider_builder=MockProviderBuilder(provider, region="us-west-1")
)
stubber = Stubber(action.s3_conn)
stubber.add_client_error(
"head_bucket",
service_error_code="NoSuchBucket",
service_message="Not Found",
http_status_code=404,
)
stubber.add_response(
"create_bucket",
service_response={},
expected_params={
"Bucket": ANY,
"CreateBucketConfiguration": {
"LocationConstraint": "us-west-1",
}
}
)
with stubber:
action.ensure_cfn_bucket()
def test_persistent_graph_no_object(self):
"""Create object if one does not exist and return empty Graph."""
context = Context(config=self.persist_graph_config)
context._s3_bucket_verified = True
stubber = Stubber(context.s3_conn)
expected_get_params = {"ResponseContentType": "application/json"}
expected_get_params.update(context.persistent_graph_location)
expected_put_params = {
"Body": "{}",
"ServerSideEncryption": "AES256",
"ACL": "bucket-owner-full-control",
"ContentType": "application/json",
}
expected_put_params.update(context.persistent_graph_location)
stubber.add_client_error("get_object",
"NoSuchKey",
expected_params=expected_get_params)
stubber.add_response("put_object", {}, expected_put_params)
with stubber:
self.assertIsNone(context._persistent_graph)
self.assertIsInstance(context.persistent_graph, Graph)
self.assertIsInstance(context._persistent_graph, Graph)
self.assertEqual({}, context.persistent_graph.to_dict())
stubber.assert_no_pending_responses()
def test_conditional_update_failed(mocker):
dynamo = boto3.resource("dynamodb")
mock_table = dynamo.Table("MOCK_TABLE")
stubber = Stubber(mock_table.meta.client)
stubber.add_client_error(
"update_item",
service_error_code="ConditionalCheckFailedException",
)
stubber.add_client_error("update_item", service_error_code="AnyOtherError")
stubber.activate()
mocker.patch.object(status, "status_table", mock_table)
# Expect no exception for ClientError code
# ConditionalCheckFailedException
status.set_pipeline_status(
"zip_id",
status.PipelineStatus.SENT_TO_UPLOADER,
)
# Expect exception for any other ClientError code
with pytest.raises(ClientError):
status.set_pipeline_status(
"zip_id",
status.PipelineStatus.SENT_TO_UPLOADER,
)
def test__create_bucket_fails(self, mock_consul):
"""
Test s3 provisioning fails on bucket creation, and retries up to 4 times
"""
instance = OpenEdXInstanceFactory()
instance.s3_access_key = 'test'
instance.s3_secret_access_key = 'test'
instance.s3_bucket_name = 'test'
max_tries = 4
stubber = Stubber(s3_client)
for _ in range(max_tries):
stubber.add_client_error('create_bucket')
with self.assertLogs('instance.models.instance', level='INFO') as cm:
with stubber:
with self.assertRaises(ClientError):
instance._create_bucket(max_tries=max_tries)
base_log_text = (
'INFO:instance.models.instance:instance={} ({!s:.15}) | Retrying bucket creation'
' due to "", attempt %s of {}.'.format(instance.ref.pk,
instance.ref.name,
max_tries))
self.assertEqual(
cm.output,
[base_log_text % i for i in range(1, max_tries + 1)])
def mock_cognito_create_user_set_mfa_fails(admin_user, create_user_arguments):
_keep_it_real()
client = boto3.real_client("cognito-idp")
group_name = admin_user["group"]["value"]
stubber = Stubber(client)
# Add responses
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_create_user(stubber, admin_user, create_user_arguments)
stub_response_cognito_list_user_pools(stubber)
# Replace admin_set_user_mfa_preference response with ClientError
# stub_response_cognito_admin_set_user_mfa_preference(stubber, admin_user["email"])
stubber.add_client_error(
"admin_set_user_mfa_preference",
expected_params={
"SMSMfaSettings": {"Enabled": True, "PreferredMfa": True},
"UserPoolId": MOCK_COGNITO_USER_POOL_ID,
"Username": admin_user["email"],
},
)
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_set_user_settings(stubber, admin_user["email"])
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_add_user_to_group(
stubber, admin_user["email"], group_name
)
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_disable_user(stubber, admin_user["email"])
stubber.activate()
# override boto.client to return the mock client
boto3.client = lambda service, region_name=None: client
return stubber
def mock_delete_user_failure(admin_user, admin_get_user):
_keep_it_real()
client = boto3.real_client("cognito-idp")
stubber = Stubber(client)
# Add responses
# get user
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_get_user(stubber, admin_user["email"], admin_get_user)
stub_response_cognito_list_user_pools(stubber)
stub_response_cognito_admin_list_groups_for_user(stubber, admin_user["email"])
# delete user
stub_response_cognito_list_user_pools(stubber)
stubber.add_client_error(
"admin_delete_user",
expected_params={
"UserPoolId": MOCK_COGNITO_USER_POOL_ID,
"Username": admin_user["email"],
},
)
stubber.activate()
# override boto.client to return the mock client
boto3.client = lambda service, region_name=None: client
return stubber
def test_list_ecr_images_repository_not_found() -> None:
"""Test list_ecr_images RepositoryNotFoundException."""
client = boto3.client("ecr")
stubber = Stubber(client)
stubber.add_client_error("list_images", "RepositoryNotFoundException")
with stubber:
assert list_ecr_images(client, repository_name="test-repo") == []
def test_get_account_name_exception():
organization = Organizations()
account_id = "12345"
client_stubber = Stubber(organization.org_client)
client_stubber.add_client_error("describe_account", "Invalid_request")
client_stubber.activate()
response = organization.get_account_name(account_id)
assert response is None
def test_create_volume_with_exception(self, mock_sleep):
client = boto3.client('ec2')
stubber = Stubber(client)
stubber.add_client_error('create_volume', service_error_code="401")
stubber.activate()
ebspin_ec2 = ec2.Ec2(client)
with self.assertRaises(botocore.exceptions.ClientError):
ebspin_ec2.create_volume(10, "gp2", "ap-southeast-2a", "foo")
def test_raises_boto3_exception(self):
client = boto3.client('ec2')
stubber = Stubber(client)
stubber.add_client_error('describe_volumes', service_error_code="401")
stubber.activate()
ebspin_ec2 = ec2.Ec2(client)
with self.assertRaises(botocore.exceptions.ClientError):
ebspin_ec2.get_latest_volume_id_available("foobar")
def test_handler_success(
sm_describe_endpoint_config_params, sm_create_endpoint_config_params,
sm_create_endpoint_config_response, cp_expected_params,
sm_describe_endpoint_params, sm_create_endpoint_params,
sm_create_endpoint_response, sm_describe_endpoint_response_2, event):
sm_client = get_client("sagemaker")
cp_client = get_client("codepipeline")
sm_stubber = Stubber(sm_client)
cp_stubber = Stubber(cp_client)
# endpoint config creation
sm_describe_endpoint_config_response = {}
cp_response = {}
sm_stubber.add_client_error(
"describe_endpoint_config",
service_error_code="EndpointConfigExists",
service_message="Could not find endpoint configuration",
http_status_code=400,
expected_params=sm_describe_endpoint_config_params,
)
sm_stubber.add_response(
"create_endpoint_config",
sm_create_endpoint_config_response,
sm_create_endpoint_config_params,
)
# endpoint creation
sm_stubber.add_client_error(
"describe_endpoint",
service_error_code="EndpointExists",
service_message="Could not find endpoint",
http_status_code=400,
expected_params=sm_describe_endpoint_params,
)
sm_stubber.add_response("create_endpoint", sm_create_endpoint_response,
sm_create_endpoint_params)
sm_stubber.add_response(
"describe_endpoint",
sm_describe_endpoint_response_2,
sm_describe_endpoint_params,
)
cp_stubber.add_response("put_job_success_result", cp_response,
cp_expected_params)
expected_log_message = (
"Sent success message back to codepipeline with job_id: test_job_id")
with sm_stubber:
with cp_stubber:
handler(event, {})
cp_stubber.assert_no_pending_responses()
reset_client()
def test_get_ou_path_exception(mocker):
organization = Organizations()
account_id = "12345"
mocker.patch.object(organization, "get_ou_name")
organization.get_ou_name.return_value = "ou-name"
client_stubber = Stubber(organization.org_client)
client_stubber.add_client_error("list_parents", "Invalid_request")
client_stubber.activate()
return_value = organization.get_ou_path(account_id)
assert return_value is None
class GlueStubber(object):
def __init__(self):
self.client = boto3.client("glue")
self.stubber = Stubber(self.client)
def add_response_for_method(self, method_name, response_body, request_params):
self.stubber.add_response(method_name, response_body, request_params)
def add_error_for_method(self, method_name, error_code, error_message, http_code, request_params):
self.stubber.add_client_error(method_name, error_code, error_message, http_code, None, request_params)
def _get_cf_stubbed_client_with_error(method: str, service_error_code: str, service_message: str = None):
client = boto3.client('cloudformation', region_name=TEST_STUB_REGION)
cf_stubber = Stubber(client)
cf_stubber.add_client_error(
method=method,
service_error_code=service_error_code,
service_message=service_message
)
cf_stubber.activate()
return client
class TestLogsCommandContext_get_resource_id_from_stack(TestCase):
def setUp(self):
self.real_client = botocore.session.get_session().create_client('cloudformation', region_name="us-east-1")
self.cfn_client_stubber = Stubber(self.real_client)
self.logical_id = "name"
self.stack_name = "stackname"
self.physical_id = "myid"
def test_must_get_from_cfn(self):
expected_params = {
"StackName": self.stack_name,
"LogicalResourceId": self.logical_id
}
mock_response = {
"StackResourceDetail": {
"PhysicalResourceId": self.physical_id,
"LogicalResourceId": self.logical_id,
"ResourceType": "AWS::Lambda::Function",
"ResourceStatus": "UPDATE_COMPLETE",
"LastUpdatedTimestamp": "2017-07-28T23:34:13.435Z"
}
}
self.cfn_client_stubber.add_response("describe_stack_resource", mock_response, expected_params)
with self.cfn_client_stubber:
result = LogsCommandContext._get_resource_id_from_stack(self.real_client,
self.stack_name,
self.logical_id)
self.assertEquals(result, self.physical_id)
def test_must_handle_resource_not_found(self):
errmsg = "Something went wrong"
errcode = "SomeException"
self.cfn_client_stubber.add_client_error("describe_stack_resource",
service_error_code=errcode,
service_message=errmsg)
expected_error_msg = "An error occurred ({}) when calling the DescribeStackResource operation: {}".format(
errcode, errmsg)
with self.cfn_client_stubber:
with self.assertRaises(UserException) as context:
LogsCommandContext._get_resource_id_from_stack(self.real_client,
self.stack_name,
self.logical_id)
self.assertEquals(expected_error_msg, str(context.exception))
def test_ensure_cfn_forbidden(self):
session = get_session("us-west-1")
provider = Provider(session)
action = BaseAction(
context=mock_context("mynamespace"),
provider_builder=MockProviderBuilder(provider)
)
stubber = Stubber(action.s3_conn)
stubber.add_client_error(
"head_bucket",
service_error_code="AccessDenied",
service_message="Forbidden",
http_status_code=403,
)
with stubber:
with self.assertRaises(botocore.exceptions.ClientError):
action.ensure_cfn_bucket()
class TestDeployer(unittest.TestCase):
def setUp(self):
client = botocore.session.get_session().create_client('cloudformation',
region_name="us-east-1")
self.stub_client = Stubber(client)
self.changeset_prefix = "some-changeset-prefix"
self.deployer = Deployer(client, self.changeset_prefix)
def test_has_stack_success(self):
stack_name = "stack_name"
expected_params = {
"StackName": stack_name
}
response = {
"Stacks": [
make_stack_obj(stack_name)
]
}
self.stub_client.add_response('describe_stacks', response,
expected_params)
with self.stub_client:
response = self.deployer.has_stack(stack_name)
self.assertTrue(response)
def test_has_stack_no_stack(self):
stack_name = "stack_name"
expected_params = {
"StackName": stack_name
}
# Response contains NO stack
no_stack_response = {
"Stacks": []
}
self.stub_client.add_response('describe_stacks', no_stack_response,
expected_params)
with self.stub_client:
response = self.deployer.has_stack(stack_name)
self.assertFalse(response)
# Response is a ClientError with a message that the stack does not exist
self.stub_client.add_client_error('describe_stacks', "ClientError",
"Stack with id {0} does not exist"
.format(stack_name))
with self.stub_client:
response = self.deployer.has_stack(stack_name)
self.assertFalse(response)
def test_has_stack_review_in_progress(self):
stack_name = "stack_name"
expected_params = {
"StackName": stack_name
}
# Response contains NO stack
review_in_progress_response = {
"Stacks": [make_stack_obj(stack_name, "REVIEW_IN_PROGRESS")]
}
self.stub_client.add_response('describe_stacks',
review_in_progress_response,
expected_params)
with self.stub_client:
response = self.deployer.has_stack(stack_name)
self.assertFalse(response)
def test_has_stack_exception(self):
self.stub_client.add_client_error('describe_stacks', "ValidationError",
"Service is bad")
with self.stub_client:
with self.assertRaises(botocore.exceptions.ClientError):
self.deployer.has_stack("stack_name")
def test_create_changeset_success(self):
stack_name = "stack_name"
template = "template"
parameters = [{"ParameterKey": "Key1", "ParameterValue": "Value",
"UsePreviousValue": True}]
capabilities = ["capabilities"]
# Case 1: Stack DOES NOT exist
self.deployer.has_stack = Mock()
self.deployer.has_stack.return_value = False
expected_params = {
"ChangeSetName": botocore.stub.ANY,
"StackName": stack_name,
"TemplateBody": template,
"ChangeSetType": "CREATE",
"Parameters": parameters,
"Capabilities": capabilities,
"Description": botocore.stub.ANY
}
response = {
"Id": "changeset ID"
}
self.stub_client.add_response("create_change_set", response,
expected_params)
with self.stub_client:
result = self.deployer.create_changeset(
stack_name, template, parameters, capabilities)
self.assertEquals(response["Id"], result.changeset_id)
self.assertEquals("CREATE", result.changeset_type)
# Case 2: Stack exists. We are updating it
self.deployer.has_stack.return_value = True
expected_params["ChangeSetType"] = "UPDATE"
self.stub_client.add_response("create_change_set", response,
expected_params)
with self.stub_client:
result = self.deployer.create_changeset(
stack_name, template, parameters, capabilities)
self.assertEquals(response["Id"], result.changeset_id)
self.assertEquals("UPDATE", result.changeset_type)
def test_create_changeset_exception(self):
stack_name = "stack_name"
template = "template"
parameters = [{"ParameterKey": "Key1", "ParameterValue": "Value",
"UsePreviousValue": True}]
capabilities = ["capabilities"]
self.deployer.has_stack = Mock()
self.deployer.has_stack.return_value = False
self.stub_client.add_client_error(
'create_change_set', "Somethign is wrong", "Service is bad")
with self.stub_client:
with self.assertRaises(botocore.exceptions.ClientError):
self.deployer.create_changeset(stack_name, template, parameters, capabilities)
def test_execute_changeset(self):
stack_name = "stack_name"
changeset_id = "changeset_id"
expected_params = {
"ChangeSetName": changeset_id,
"StackName": stack_name
}
self.stub_client.add_response("execute_change_set", {}, expected_params)
with self.stub_client:
self.deployer.execute_changeset(changeset_id, stack_name)
def test_execute_changeset_exception(self):
stack_name = "stack_name"
changeset_id = "changeset_id"
self.stub_client.add_client_error(
'execute_change_set', "Somethign is wrong", "Service is bad")
with self.stub_client:
with self.assertRaises(botocore.exceptions.ClientError):
self.deployer.execute_changeset(changeset_id, stack_name)
def test_create_and_wait_for_changeset_successful(self):
stack_name = "stack_name"
template = "template"
parameters = [{"ParameterKey": "Key1", "ParameterValue": "Value",
"UsePreviousValue": True}]
capabilities = ["capabilities"]
changeset_id = "changeset id"
changeset_type = "changeset type"
self.deployer.create_changeset = Mock()
self.deployer.create_changeset.return_value = ChangeSetResult(changeset_id, changeset_type)
self.deployer.wait_for_changeset = Mock()
result = self.deployer.create_and_wait_for_changeset(
stack_name, template, parameters, capabilities)
self.assertEquals(result.changeset_id, changeset_id)
self.assertEquals(result.changeset_type, changeset_type)
def test_create_and_wait_for_changeset_error_waiting_for_changeset(self):
stack_name = "stack_name"
template = "template"
parameters = [{"ParameterKey": "Key1", "ParameterValue": "Value",
"UsePreviousValue": True}]
capabilities = ["capabilities"]
changeset_id = "changeset id"
changeset_type = "changeset type"
self.deployer.create_changeset = Mock()
self.deployer.create_changeset.return_value = ChangeSetResult(changeset_id, changeset_type)
self.deployer.wait_for_changeset = Mock()
self.deployer.wait_for_changeset.side_effect = RuntimeError
with self.assertRaises(RuntimeError):
result = self.deployer.create_and_wait_for_changeset(
stack_name, template, parameters, capabilities)
def test_wait_for_changeset_no_changes(self):
stack_name = "stack_name"
changeset_id = "changeset-id"
mock_client = Mock()
mock_deployer = Deployer(mock_client)
mock_waiter = Mock()
mock_client.get_waiter.return_value = mock_waiter
response = {
"Status": "FAILED",
"StatusReason": "No updates are to be performed"
}
waiter_error = botocore.exceptions.WaiterError(name="name",
reason="reason",
last_response=response)
mock_waiter.wait.side_effect = waiter_error
with self.assertRaises(exceptions.ChangeEmptyError):
mock_deployer.wait_for_changeset(changeset_id, stack_name)
mock_waiter.wait.assert_called_once_with(ChangeSetName=changeset_id,
StackName=stack_name)
mock_client.get_waiter.assert_called_once_with(
"change_set_create_complete")
def test_wait_for_changeset_failed_to_create_changeset(self):
stack_name = "stack_name"
changeset_id = "changeset-id"
mock_client = Mock()
mock_deployer = Deployer(mock_client)
mock_waiter = Mock()
mock_client.get_waiter.return_value = mock_waiter
response = {
"Status": "FAILED",
"StatusReason": "some reason"
}
waiter_error = botocore.exceptions.WaiterError(name="name",
reason="reason",
last_response=response)
mock_waiter.wait.side_effect = waiter_error
with self.assertRaises(RuntimeError):
mock_deployer.wait_for_changeset(changeset_id, stack_name)
mock_waiter.wait.assert_called_once_with(ChangeSetName=changeset_id,
StackName=stack_name)
mock_client.get_waiter.assert_called_once_with(
"change_set_create_complete")
def test_wait_for_execute_no_changes(self):
stack_name = "stack_name"
changeset_type = "CREATE"
mock_client = Mock()
mock_deployer = Deployer(mock_client)
mock_waiter = Mock()
mock_client.get_waiter.return_value = mock_waiter
waiter_error = botocore.exceptions.WaiterError(name="name",
reason="reason",
last_response={})
mock_waiter.wait.side_effect = waiter_error
with self.assertRaises(exceptions.DeployFailedError):
mock_deployer.wait_for_execute(stack_name, changeset_type)
mock_waiter.wait.assert_called_once_with(StackName=stack_name)
mock_client.get_waiter.assert_called_once_with(
"stack_create_complete")
class TestStubber(unittest.TestCase):
def setUp(self):
session = botocore.session.get_session()
config = botocore.config.Config(signature_version=botocore.UNSIGNED)
self.client = session.create_client('s3', config=config)
self.stubber = Stubber(self.client)
def test_stubber_returns_response(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
self.stubber.add_response('list_objects', service_response)
self.stubber.activate()
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_context_manager_returns_response(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
self.stubber.add_response('list_objects', service_response)
with self.stubber:
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_activated_stubber_errors_with_no_registered_stubs(self):
self.stubber.activate()
# Params one per line for readability.
with self.assertRaisesRegexp(StubResponseError,
"'Bucket': 'asdfasdfasdfasdf',\n"):
self.client.list_objects(
Bucket='asdfasdfasdfasdf',
Delimiter='asdfasdfasdfasdf',
Prefix='asdfasdfasdfasdf',
EncodingType='url')
def test_stubber_errors_when_stubs_are_used_up(self):
self.stubber.add_response('list_objects', {})
self.stubber.activate()
self.client.list_objects(Bucket='foo')
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
def test_client_error_response(self):
error_code = "AccessDenied"
error_message = "Access Denied"
self.stubber.add_client_error(
'list_objects', error_code, error_message)
self.stubber.activate()
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
def test_expected_params_success(self):
service_response = {}
expected_params = {'Bucket': 'foo'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# This should be called successfully with no errors being thrown
# for mismatching expected params.
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_expected_params_fail(self):
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# This should call should raise an for mismatching expected params.
with self.assertRaisesRegexp(StubResponseError,
"{'Bucket': 'bar'},\n"):
self.client.list_objects(Bucket='foo')
def test_expected_params_mixed_with_errors_responses(self):
# Add an error response
error_code = "AccessDenied"
error_message = "Access Denied"
self.stubber.add_client_error(
'list_objects', error_code, error_message)
# Add a response with incorrect expected params
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# The first call should throw and error as expected.
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
# The second call should throw an error for unexpected parameters
with self.assertRaisesRegexp(StubResponseError, 'Expected parameters'):
self.client.list_objects(Bucket='foo')
def test_can_continue_to_call_after_expected_params_fail(self):
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for unexpected parameters
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
# The stubber should still have the responses queued up
# even though the original parameters did not match the expected ones.
self.client.list_objects(Bucket='bar')
self.stubber.assert_no_pending_responses()
def test_still_relies_on_param_validation_with_expected_params(self):
service_response = {}
expected_params = {'Buck': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for invalid parameters
with self.assertRaises(ParamValidationError):
self.client.list_objects(Buck='bar')
def test_any_ignores_param_for_validation(self):
service_response = {}
expected_params = {'Bucket': stub.ANY}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.add_response(
'list_objects', service_response, expected_params)
try:
with self.stubber:
self.client.list_objects(Bucket='foo')
self.client.list_objects(Bucket='bar')
except StubAssertionError:
self.fail("stub.ANY failed to ignore parameter for validation.")
def test_mixed_any_and_concrete_params(self):
service_response = {}
expected_params = {'Bucket': stub.ANY, 'Key': 'foo.txt'}
self.stubber.add_response(
'head_object', service_response, expected_params)
self.stubber.add_response(
'head_object', service_response, expected_params)
try:
with self.stubber:
self.client.head_object(Bucket='foo', Key='foo.txt')
self.client.head_object(Bucket='bar', Key='foo.txt')
except StubAssertionError:
self.fail("stub.ANY failed to ignore parameter for validation.")
def test_none_param(self):
service_response = {}
expected_params = {'Buck': None}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for invalid parameters
with self.assertRaises(StubAssertionError):
self.client.list_objects(Buck='bar')
def test_many_expected_params(self):
service_response = {}
expected_params = {
'Bucket': 'mybucket',
'Prefix': 'myprefix',
'Delimiter': '/',
'EncodingType': 'url'
}
self.stubber.add_response(
'list_objects', service_response, expected_params)
try:
with self.stubber:
self.client.list_objects(**expected_params)
except StubAssertionError:
self.fail(
"Stubber inappropriately raised error for same parameters.")
class TestDynamoDBHandler(unittest.TestCase):
client = boto3.client('dynamodb', region_name='us-east-1')
def setUp(self):
self.stubber = Stubber(self.client)
self.get_parameters_response = {'Item': {'TestMap': {'M': {
'String1': {'S': 'StringVal1'},
'List1': {'L': [
{'S': 'ListVal1'},
{'S': 'ListVal2'}]},
'Number1': {'N': '12345'}, }}}}
@mock.patch('stacker.lookups.handlers.dynamodb.get_session',
return_value=SessionStub(client))
def test_dynamodb_handler(self, mock_client):
expected_params = {
'TableName': 'TestTable',
'Key': {
'TestKey': {'S': 'TestVal'}
},
'ProjectionExpression': 'TestVal,TestMap,String1'
}
base_lookup_key = 'TestTable@TestKey:TestVal.TestMap[M].String1'
base_lookup_key_valid = 'StringVal1'
self.stubber.add_response('get_item',
self.get_parameters_response,
expected_params)
with self.stubber:
value = DynamodbLookup.handle(base_lookup_key)
self.assertEqual(value, base_lookup_key_valid)
@mock.patch('stacker.lookups.handlers.dynamodb.get_session',
return_value=SessionStub(client))
def test_dynamodb_number_handler(self, mock_client):
expected_params = {
'TableName': 'TestTable',
'Key': {
'TestKey': {'S': 'TestVal'}
},
'ProjectionExpression': 'TestVal,TestMap,Number1'
}
base_lookup_key = 'TestTable@TestKey:TestVal.' \
'TestMap[M].Number1[N]'
base_lookup_key_valid = 12345
self.stubber.add_response('get_item',
self.get_parameters_response,
expected_params)
with self.stubber:
value = DynamodbLookup.handle(base_lookup_key)
self.assertEqual(value, base_lookup_key_valid)
@mock.patch('stacker.lookups.handlers.dynamodb.get_session',
return_value=SessionStub(client))
def test_dynamodb_list_handler(self, mock_client):
expected_params = {
'TableName': 'TestTable',
'Key': {
'TestKey': {'S': 'TestVal'}
},
'ProjectionExpression': 'TestVal,TestMap,List1'
}
base_lookup_key = 'TestTable@TestKey:TestVal.' \
'TestMap[M].List1[L]'
base_lookup_key_valid = ['ListVal1', 'ListVal2']
self.stubber.add_response('get_item',
self.get_parameters_response,
expected_params)
with self.stubber:
value = DynamodbLookup.handle(base_lookup_key)
self.assertEqual(value, base_lookup_key_valid)
@mock.patch('stacker.lookups.handlers.dynamodb.get_session',
return_value=SessionStub(client))
def test_dynamodb_empty_table_handler(self, mock_client):
expected_params = {
'TableName': '',
'Key': {
'TestKey': {'S': 'TestVal'}
},
'ProjectionExpression': 'TestVal,TestMap,String1'
}
base_lookup_key = '@TestKey:TestVal.TestMap[M].String1'
self.stubber.add_response('get_item',
self.get_parameters_response,
expected_params)
with self.stubber:
try:
DynamodbLookup.handle(base_lookup_key)
except ValueError as e:
self.assertEqual(
'Please make sure to include a dynamodb table name',
str(e))
@mock.patch('stacker.lookups.handlers.dynamodb.get_session',
return_value=SessionStub(client))
def test_dynamodb_missing_table_handler(self, mock_client):
expected_params = {
'Key': {
'TestKey': {'S': 'TestVal'}
},
'ProjectionExpression': 'TestVal,TestMap,String1'
}
base_lookup_key = 'TestKey:TestVal.TestMap[M].String1'
self.stubber.add_response('get_item',
self.get_parameters_response,
expected_params)
with self.stubber:
try:
DynamodbLookup.handle(base_lookup_key)
except ValueError as e:
self.assertEqual(
'Please make sure to include a tablename',
str(e))
@mock.patch('stacker.lookups.handlers.dynamodb.get_session',
return_value=SessionStub(client))
def test_dynamodb_invalid_table_handler(self, mock_client):
expected_params = {
'TableName': 'FakeTable',
'Key': {
'TestKey': {'S': 'TestVal'}
},
'ProjectionExpression': 'TestVal,TestMap,String1'
}
base_lookup_key = 'FakeTable@TestKey:TestVal.TestMap[M].String1'
service_error_code = 'ResourceNotFoundException'
self.stubber.add_client_error('get_item',
service_error_code=service_error_code,
expected_params=expected_params)
with self.stubber:
try:
DynamodbLookup.handle(base_lookup_key)
except ValueError as e:
self.assertEqual(
'Cannot find the dynamodb table: FakeTable',
str(e))
@mock.patch('stacker.lookups.handlers.dynamodb.get_session',
return_value=SessionStub(client))
def test_dynamodb_invalid_partition_key_handler(self, mock_client):
expected_params = {
'TableName': 'TestTable',
'Key': {
'FakeKey': {'S': 'TestVal'}
},
'ProjectionExpression': 'TestVal,TestMap,String1'
}
base_lookup_key = 'TestTable@FakeKey:TestVal.TestMap[M].String1'
service_error_code = 'ValidationException'
self.stubber.add_client_error('get_item',
service_error_code=service_error_code,
expected_params=expected_params)
with self.stubber:
try:
DynamodbLookup.handle(base_lookup_key)
except ValueError as e:
self.assertEqual(
'No dynamodb record matched the partition key: FakeKey',
str(e))
@mock.patch('stacker.lookups.handlers.dynamodb.get_session',
return_value=SessionStub(client))
def test_dynamodb_invalid_partition_val_handler(self, mock_client):
expected_params = {
'TableName': 'TestTable',
'Key': {
'TestKey': {'S': 'FakeVal'}
},
'ProjectionExpression': 'FakeVal,TestMap,String1'
}
empty_response = {'ResponseMetadata': {}}
base_lookup_key = 'TestTable@TestKey:FakeVal.TestMap[M].String1'
self.stubber.add_response('get_item',
empty_response,
expected_params)
with self.stubber:
try:
DynamodbLookup.handle(base_lookup_key)
except ValueError as e:
self.assertEqual(
'The dynamodb record could not be found using '
'the following key: {\'S\': \'FakeVal\'}',
str(e))
class TestS3Uploader(unittest.TestCase):
def setUp(self):
self.s3client = botocore.session.get_session().create_client(
's3', region_name="us-east-1")
self.s3client_stub = Stubber(self.s3client)
self.transfer_manager_mock = Mock(spec=S3Transfer)
self.transfer_manager_mock.upload = Mock()
self.bucket_name = "bucketname"
self.prefix = None
self.region = "us-east-1"
self.s3uploader = S3Uploader(
self.s3client, self.bucket_name, self.region, self.prefix, None, False,
self.transfer_manager_mock)
@patch("awscli.customizations.cloudformation.s3uploader.ProgressPercentage")
def test_upload_successful(self, progress_percentage_mock):
file_name = "filename"
remote_path = "remotepath"
prefix = "SomePrefix"
remote_path_with_prefix = "{0}/{1}".format(prefix, remote_path)
s3uploader = S3Uploader(
self.s3client, self.bucket_name, self.region, prefix, None, False,
self.transfer_manager_mock)
expected_upload_url = "s3://{0}/{1}/{2}".format(
self.bucket_name, prefix, remote_path)
# Setup mock to fake that file does not exist
s3uploader.file_exists = Mock()
s3uploader.file_exists.return_value = False
upload_url = s3uploader.upload(file_name, remote_path)
self.assertEquals(expected_upload_url, upload_url)
expected_encryption_args = {
"ServerSideEncryption": "AES256"
}
self.transfer_manager_mock.upload.assert_called_once_with(
file_name, self.bucket_name, remote_path_with_prefix,
expected_encryption_args, mock.ANY)
s3uploader.file_exists.assert_called_once_with(remote_path_with_prefix)
@patch("awscli.customizations.cloudformation.s3uploader.ProgressPercentage")
def test_upload_idempotency(self, progress_percentage_mock):
file_name = "filename"
remote_path = "remotepath"
# Setup mock to fake that file was already uploaded
self.s3uploader.file_exists = Mock()
self.s3uploader.file_exists.return_value = True
self.s3uploader.upload(file_name, remote_path)
self.transfer_manager_mock.upload.assert_not_called()
self.s3uploader.file_exists.assert_called_once_with(remote_path)
@patch("awscli.customizations.cloudformation.s3uploader.ProgressPercentage")
def test_upload_force_upload(self, progress_percentage_mock):
file_name = "filename"
remote_path = "remotepath"
expected_upload_url = "s3://{0}/{1}".format(self.bucket_name,
remote_path)
# Set ForceUpload = True
self.s3uploader = S3Uploader(
self.s3client, self.bucket_name, self.region, self.prefix,
None, True, self.transfer_manager_mock)
# Pretend file already exists
self.s3uploader.file_exists = Mock()
self.s3uploader.file_exists.return_value = True
# Because we forced an update, this should reupload even if file exists
upload_url = self.s3uploader.upload(file_name, remote_path)
self.assertEquals(expected_upload_url, upload_url)
expected_encryption_args = {
"ServerSideEncryption": "AES256"
}
self.transfer_manager_mock.upload.assert_called_once_with(
file_name, self.bucket_name, remote_path,
expected_encryption_args, mock.ANY)
# Since ForceUpload=True, we should NEVER do the file-exists check
self.s3uploader.file_exists.assert_not_called()
@patch("awscli.customizations.cloudformation.s3uploader.ProgressPercentage")
def test_upload_successful_custom_kms_key(self, progress_percentage_mock):
file_name = "filename"
remote_path = "remotepath"
kms_key_id = "kms_id"
expected_upload_url = "s3://{0}/{1}".format(self.bucket_name,
remote_path)
# Set KMS Key Id
self.s3uploader = S3Uploader(
self.s3client, self.bucket_name, self.region, self.prefix,
kms_key_id, False, self.transfer_manager_mock)
# Setup mock to fake that file does not exist
self.s3uploader.file_exists = Mock()
self.s3uploader.file_exists.return_value = False
upload_url = self.s3uploader.upload(file_name, remote_path)
self.assertEquals(expected_upload_url, upload_url)
expected_encryption_args = {
"ServerSideEncryption": "aws:kms",
"SSEKMSKeyId": kms_key_id
}
self.transfer_manager_mock.upload.assert_called_once_with(
file_name, self.bucket_name, remote_path,
expected_encryption_args, mock.ANY)
self.s3uploader.file_exists.assert_called_once_with(remote_path)
@patch("awscli.customizations.cloudformation.s3uploader.ProgressPercentage")
def test_upload_successful_nobucket(self, progress_percentage_mock):
file_name = "filename"
remote_path = "remotepath"
# Setup mock to fake that file does not exist
self.s3uploader.file_exists = Mock()
self.s3uploader.file_exists.return_value = False
# Setup uploader to return a NOSuchBucket exception
exception = botocore.exceptions.ClientError(
{"Error": {"Code": "NoSuchBucket"}}, "OpName")
self.transfer_manager_mock.upload.side_effect = exception
with self.assertRaises(exceptions.NoSuchBucketError):
self.s3uploader.upload(file_name, remote_path)
@patch("awscli.customizations.cloudformation.s3uploader.ProgressPercentage")
def test_upload_successful_exceptions(self, progress_percentage_mock):
file_name = "filename"
remote_path = "remotepath"
# Setup mock to fake that file does not exist
self.s3uploader.file_exists = Mock()
self.s3uploader.file_exists.return_value = False
# Raise an unrecognized botocore error
exception = botocore.exceptions.ClientError(
{"Error": {"Code": "SomeError"}}, "OpName")
self.transfer_manager_mock.upload.side_effect = exception
with self.assertRaises(botocore.exceptions.ClientError):
self.s3uploader.upload(file_name, remote_path)
# Some other exception
self.transfer_manager_mock.upload.side_effect = FloatingPointError()
with self.assertRaises(FloatingPointError):
self.s3uploader.upload(file_name, remote_path)
def test_upload_with_dedup(self):
checksum = "some md5 checksum"
filename = "filename"
extension = "extn"
self.s3uploader.file_checksum = Mock()
self.s3uploader.file_checksum.return_value = checksum
self.s3uploader.upload = Mock()
self.s3uploader.upload_with_dedup(filename, extension)
remotepath = "{0}.{1}".format(checksum, extension)
self.s3uploader.upload.assert_called_once_with(filename, remotepath)
def test_file_exists(self):
key = "some/path"
expected_params = {
"Bucket": self.bucket_name,
"Key": key
}
response = {
"AcceptRanges": "bytes",
"ContentType": "text/html",
"LastModified": "Thu, 16 Apr 2015 18:19:14 GMT",
"ContentLength": 77,
"VersionId": "null",
"ETag": "\"30a6ec7e1a9ad79c203d05a589c8b400\"",
"Metadata": {}
}
# Let's pretend file exists
self.s3client_stub.add_response("head_object",
response,
expected_params)
with self.s3client_stub:
self.assertTrue(self.s3uploader.file_exists(key))
# Let's pretend file does not exist
self.s3client_stub.add_client_error(
'head_object', "ClientError", "some error")
with self.s3client_stub:
self.assertFalse(self.s3uploader.file_exists(key))
# Let's pretend some other unknown exception happened
s3mock = Mock()
uploader = S3Uploader(s3mock, self.bucket_name, self.region)
s3mock.head_object = Mock()
s3mock.head_object.side_effect = RuntimeError()
with self.assertRaises(RuntimeError):
uploader.file_exists(key)
def test_file_checksum(self):
num_chars = 4096*5
data = ''.join(random.choice(string.ascii_uppercase)
for _ in range(num_chars)).encode('utf-8')
md5 = hashlib.md5()
md5.update(data)
expected_checksum = md5.hexdigest()
tempdir = tempfile.mkdtemp()
try:
filename = os.path.join(tempdir, 'tempfile')
with open(filename, 'wb') as f:
f.write(data)
actual_checksum = self.s3uploader.file_checksum(filename)
self.assertEqual(expected_checksum, actual_checksum)
finally:
shutil.rmtree(tempdir)
def test_make_url(self):
path = "Hello/how/are/you"
expected = "s3://{0}/{1}".format(self.bucket_name, path)
self.assertEquals(expected, self.s3uploader.make_url(path))
def test_to_path_style_s3_url_us_east_1(self):
key = "path/to/file"
version = "someversion"
region = "us-east-1"
s3uploader = S3Uploader(self.s3client, self.bucket_name, region)
result = s3uploader.to_path_style_s3_url(key, version)
self.assertEqual(
result,
"https://s3.amazonaws.com/{0}/{1}?versionId={2}".format(
self.bucket_name, key, version))
# Without versionId, that query parameter should be omitted
s3uploader = S3Uploader(self.s3client, self.bucket_name, region)
result = s3uploader.to_path_style_s3_url(key)
self.assertEqual(
result,
"https://s3.amazonaws.com/{0}/{1}".format(
self.bucket_name, key, version))
def test_to_path_style_s3_url_other_regions(self):
key = "path/to/file"
version = "someversion"
region = "us-west-2"
s3uploader = S3Uploader(self.s3client, self.bucket_name, region)
result = s3uploader.to_path_style_s3_url(key, version)
self.assertEqual(
result,
"https://s3-{0}.amazonaws.com/{1}/{2}?versionId={3}".format(
region, self.bucket_name, key, version))
# Without versionId, that query parameter should be omitted
s3uploader = S3Uploader(self.s3client, self.bucket_name, region)
result = s3uploader.to_path_style_s3_url(key)
self.assertEqual(
result,
"https://s3-{0}.amazonaws.com/{1}/{2}".format(
region, self.bucket_name, key))
class TestProviderDefaultMode(unittest.TestCase):
def setUp(self):
region = "us-east-1"
self.session = get_session(region=region)
self.provider = Provider(
self.session, region=region, recreate_failed=False)
self.stubber = Stubber(self.provider.cloudformation)
def test_get_stack_stack_does_not_exist(self):
stack_name = "MockStack"
self.stubber.add_client_error(
"describe_stacks",
service_error_code="ValidationError",
service_message="Stack with id %s does not exist" % stack_name,
expected_params={"StackName": stack_name}
)
with self.assertRaises(exceptions.StackDoesNotExist):
with self.stubber:
self.provider.get_stack(stack_name)
def test_get_stack_stack_exists(self):
stack_name = "MockStack"
stack_response = {
"Stacks": [generate_describe_stacks_stack(stack_name)]
}
self.stubber.add_response(
"describe_stacks",
stack_response,
expected_params={"StackName": stack_name}
)
with self.stubber:
response = self.provider.get_stack(stack_name)
self.assertEqual(response["StackName"], stack_name)
def test_select_update_method(self):
for i in [[{'force_interactive': True,
'force_change_set': False},
self.provider.interactive_update_stack],
[{'force_interactive': False,
'force_change_set': False},
self.provider.default_update_stack],
[{'force_interactive': False,
'force_change_set': True},
self.provider.noninteractive_changeset_update],
[{'force_interactive': True,
'force_change_set': True},
self.provider.interactive_update_stack]]:
self.assertEquals(
self.provider.select_update_method(**i[0]),
i[1]
)
def test_prepare_stack_for_update_completed(self):
stack_name = "MockStack"
stack = generate_describe_stacks_stack(
stack_name, stack_status="UPDATE_COMPLETE")
with self.stubber:
self.assertTrue(
self.provider.prepare_stack_for_update(stack, []))
def test_prepare_stack_for_update_in_progress(self):
stack_name = "MockStack"
stack = generate_describe_stacks_stack(
stack_name, stack_status="UPDATE_IN_PROGRESS")
with self.assertRaises(exceptions.StackUpdateBadStatus) as raised:
with self.stubber:
self.provider.prepare_stack_for_update(stack, [])
self.assertIn('in-progress', str(raised.exception))
def test_prepare_stack_for_update_non_recreatable(self):
stack_name = "MockStack"
stack = generate_describe_stacks_stack(
stack_name, stack_status="REVIEW_IN_PROGRESS")
with self.assertRaises(exceptions.StackUpdateBadStatus) as raised:
with self.stubber:
self.provider.prepare_stack_for_update(stack, [])
self.assertIn('Unsupported state', str(raised.exception))
def test_prepare_stack_for_update_disallowed(self):
stack_name = "MockStack"
stack = generate_describe_stacks_stack(
stack_name, stack_status="ROLLBACK_COMPLETE")
with self.assertRaises(exceptions.StackUpdateBadStatus) as raised:
with self.stubber:
self.provider.prepare_stack_for_update(stack, [])
self.assertIn('re-creation is disabled', str(raised.exception))
# Ensure we point out to the user how to enable re-creation
self.assertIn('--recreate-failed', str(raised.exception))
def test_prepare_stack_for_update_bad_tags(self):
stack_name = "MockStack"
stack = generate_describe_stacks_stack(
stack_name, stack_status="ROLLBACK_COMPLETE")
self.provider.recreate_failed = True
with self.assertRaises(exceptions.StackUpdateBadStatus) as raised:
with self.stubber:
self.provider.prepare_stack_for_update(
stack,
tags=[{'Key': 'stacker_namespace', 'Value': 'test'}])
self.assertIn('tags differ', str(raised.exception).lower())
def test_prepare_stack_for_update_recreate(self):
stack_name = "MockStack"
stack = generate_describe_stacks_stack(
stack_name, stack_status="ROLLBACK_COMPLETE")
self.stubber.add_response(
"delete_stack",
{},
expected_params={"StackName": stack_name}
)
self.provider.recreate_failed = True
with self.stubber:
self.assertFalse(
self.provider.prepare_stack_for_update(stack, []))
def test_noninteractive_changeset_update_no_stack_policy(self):
stack_name = "MockStack"
self.stubber.add_response(
"create_change_set",
{'Id': 'CHANGESETID', 'StackId': 'STACKID'}
)
changes = []
changes.append(generate_change())
self.stubber.add_response(
"describe_change_set",
generate_change_set_response(
status="CREATE_COMPLETE", execution_status="AVAILABLE",
changes=changes,
)
)
self.stubber.add_response("execute_change_set", {})
with self.stubber:
self.provider.noninteractive_changeset_update(
fqn=stack_name,
template=Template(url="http://fake.template.url.com/"),
old_parameters=[],
parameters=[], stack_policy=None, tags=[],
)
def test_noninteractive_changeset_update_with_stack_policy(self):
stack_name = "MockStack"
self.stubber.add_response(
"create_change_set",
{'Id': 'CHANGESETID', 'StackId': 'STACKID'}
)
changes = []
changes.append(generate_change())
self.stubber.add_response(
"describe_change_set",
generate_change_set_response(
status="CREATE_COMPLETE", execution_status="AVAILABLE",
changes=changes,
)
)
self.stubber.add_response("set_stack_policy", {})
self.stubber.add_response("execute_change_set", {})
with self.stubber:
self.provider.noninteractive_changeset_update(
fqn=stack_name,
template=Template(url="http://fake.template.url.com/"),
old_parameters=[],
parameters=[], stack_policy=Template(body="{}"), tags=[],
)
def test_tail_stack_retry_on_missing_stack(self):
stack_name = "SlowToCreateStack"
stack = MagicMock(spec=Stack)
stack.fqn = "my-namespace-{}".format(stack_name)
default.TAIL_RETRY_SLEEP = .01
# Ensure the stack never appears before we run out of retries
for i in range(MAX_TAIL_RETRIES + 5):
self.stubber.add_client_error(
"describe_stack_events",
service_error_code="ValidationError",
service_message="Stack [{}] does not exist".format(stack_name),
http_status_code=400,
response_meta={"attempt": i + 1},
)
with self.stubber:
try:
self.provider.tail_stack(stack, threading.Event())
except ClientError as exc:
self.assertEqual(
exc.response["ResponseMetadata"]["attempt"],
MAX_TAIL_RETRIES
)
def test_tail_stack_retry_on_missing_stack_eventual_success(self):
stack_name = "SlowToCreateStack"
stack = MagicMock(spec=Stack)
stack.fqn = "my-namespace-{}".format(stack_name)
default.TAIL_RETRY_SLEEP = .01
default.GET_EVENTS_SLEEP = .01
rcvd_events = []
def mock_log_func(e):
rcvd_events.append(e)
def valid_event_response(stack, event_id):
return {
"StackEvents": [
{
"StackId": stack.fqn + "12345",
"EventId": event_id,
"StackName": stack.fqn,
"Timestamp": datetime.now()
},
]
}
# Ensure the stack never appears before we run out of retries
for i in range(3):
self.stubber.add_client_error(
"describe_stack_events",
service_error_code="ValidationError",
service_message="Stack [{}] does not exist".format(stack_name),
http_status_code=400,
response_meta={"attempt": i + 1},
)
self.stubber.add_response(
"describe_stack_events",
valid_event_response(stack, "InitialEvents")
)
self.stubber.add_response(
"describe_stack_events",
valid_event_response(stack, "Event1")
)
with self.stubber:
try:
self.provider.tail_stack(stack, threading.Event(),
log_func=mock_log_func)
except UnStubbedResponseError:
# Eventually we run out of responses - could not happen in
# regular execution
# normally this would just be dealt with when the threads were
# shutdown, but doing so here is a little difficult because
# we can't control the `tail_stack` loop
pass
self.assertEqual(rcvd_events[0]["EventId"], "Event1")
class TestStubber(unittest.TestCase):
def setUp(self):
session = botocore.session.get_session()
config = botocore.client.Config(signature_version=botocore.UNSIGNED)
self.client = session.create_client('s3', config=config)
self.stubber = Stubber(self.client)
def test_stubber_returns_response(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
self.stubber.add_response('list_objects', service_response)
self.stubber.activate()
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_activated_stubber_errors_with_no_registered_stubs(self):
self.stubber.activate()
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
def test_stubber_errors_when_stubs_are_used_up(self):
self.stubber.add_response('list_objects', {})
self.stubber.activate()
self.client.list_objects(Bucket='foo')
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
def test_client_error_response(self):
error_code = "AccessDenied"
error_message = "Access Denied"
self.stubber.add_client_error(
'list_objects', error_code, error_message)
self.stubber.activate()
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
def test_expected_params_success(self):
service_response = {}
expected_params = {'Bucket': 'foo'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# This should be called successfully with no errors being thrown
# for mismatching expected params.
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_expected_params_fail(self):
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# This should call should raise an for mismatching expected params.
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
def test_expected_params_mixed_with_errors_responses(self):
# Add an error response
error_code = "AccessDenied"
error_message = "Access Denied"
self.stubber.add_client_error(
'list_objects', error_code, error_message)
# Add a response with incorrect expected params
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# The first call should throw and error as expected.
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
# The second call should throw an error for unexpected parameters
with self.assertRaisesRegexp(StubResponseError, 'Expected parameters'):
self.client.list_objects(Bucket='foo')
def test_can_continue_to_call_after_expected_params_fail(self):
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for unexpected parameters
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
# The stubber should still have the responses queued up
# even though the original parameters did not match the expected ones.
self.client.list_objects(Bucket='bar')
self.stubber.assert_no_pending_responses()
def test_still_relies_on_param_validation_with_expected_params(self):
service_response = {}
expected_params = {'Buck': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for invalid parameters
with self.assertRaises(ParamValidationError):
self.client.list_objects(Buck='bar')
class TestStubber(unittest.TestCase):
def setUp(self):
session = botocore.session.get_session()
config = botocore.config.Config(
signature_version=botocore.UNSIGNED,
s3={'addressing_style': 'path'}
)
self.client = session.create_client(
's3', region_name='us-east-1', config=config)
self.stubber = Stubber(self.client)
def test_stubber_returns_response(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
self.stubber.add_response('list_objects', service_response)
self.stubber.activate()
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_context_manager_returns_response(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
self.stubber.add_response('list_objects', service_response)
with self.stubber:
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_activated_stubber_errors_with_no_registered_stubs(self):
self.stubber.activate()
# Params one per line for readability.
with self.assertRaisesRegexp(UnStubbedResponseError,
"Unexpected API Call"):
self.client.list_objects(
Bucket='asdfasdfasdfasdf',
Delimiter='asdfasdfasdfasdf',
Prefix='asdfasdfasdfasdf',
EncodingType='url')
def test_stubber_errors_when_stubs_are_used_up(self):
self.stubber.add_response('list_objects', {})
self.stubber.activate()
self.client.list_objects(Bucket='foo')
with self.assertRaises(UnStubbedResponseError):
self.client.list_objects(Bucket='foo')
def test_client_error_response(self):
error_code = "AccessDenied"
error_message = "Access Denied"
self.stubber.add_client_error(
'list_objects', error_code, error_message)
self.stubber.activate()
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
def test_can_add_expected_params_to_client_error(self):
self.stubber.add_client_error(
'list_objects', 'Error', 'error',
expected_params={'Bucket': 'foo'}
)
self.stubber.activate()
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
def test_can_expected_param_fails_in_client_error(self):
self.stubber.add_client_error(
'list_objects', 'Error', 'error',
expected_params={'Bucket': 'foo'}
)
self.stubber.activate()
# We expect an AssertionError instead of a ClientError
# because we're calling the operation with the wrong
# param value.
with self.assertRaises(AssertionError):
self.client.list_objects(Bucket='wrong-argument-value')
def test_expected_params_success(self):
service_response = {}
expected_params = {'Bucket': 'foo'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# This should be called successfully with no errors being thrown
# for mismatching expected params.
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_expected_params_fail(self):
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# This should call should raise an for mismatching expected params.
with self.assertRaisesRegexp(StubResponseError,
"{'Bucket': 'bar'},\n"):
self.client.list_objects(Bucket='foo')
def test_expected_params_mixed_with_errors_responses(self):
# Add an error response
error_code = "AccessDenied"
error_message = "Access Denied"
self.stubber.add_client_error(
'list_objects', error_code, error_message)
# Add a response with incorrect expected params
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# The first call should throw and error as expected.
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
# The second call should throw an error for unexpected parameters
with self.assertRaisesRegexp(StubResponseError, 'Expected parameters'):
self.client.list_objects(Bucket='foo')
def test_can_continue_to_call_after_expected_params_fail(self):
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for unexpected parameters
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
# The stubber should still have the responses queued up
# even though the original parameters did not match the expected ones.
self.client.list_objects(Bucket='bar')
self.stubber.assert_no_pending_responses()
def test_still_relies_on_param_validation_with_expected_params(self):
service_response = {}
expected_params = {'Buck': 'bar'}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for invalid parameters
with self.assertRaises(ParamValidationError):
self.client.list_objects(Buck='bar')
def test_any_ignores_param_for_validation(self):
service_response = {}
expected_params = {'Bucket': stub.ANY}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.add_response(
'list_objects', service_response, expected_params)
try:
with self.stubber:
self.client.list_objects(Bucket='foo')
self.client.list_objects(Bucket='bar')
except StubAssertionError:
self.fail("stub.ANY failed to ignore parameter for validation.")
def test_mixed_any_and_concrete_params(self):
service_response = {}
expected_params = {'Bucket': stub.ANY, 'Key': 'foo.txt'}
self.stubber.add_response(
'head_object', service_response, expected_params)
self.stubber.add_response(
'head_object', service_response, expected_params)
try:
with self.stubber:
self.client.head_object(Bucket='foo', Key='foo.txt')
self.client.head_object(Bucket='bar', Key='foo.txt')
except StubAssertionError:
self.fail("stub.ANY failed to ignore parameter for validation.")
def test_nested_any_param(self):
service_response = {}
expected_params = {
'Bucket': 'foo',
'Key': 'bar.txt',
'Metadata': {
'MyMeta': stub.ANY,
}
}
self.stubber.add_response(
'put_object', service_response, expected_params)
self.stubber.add_response(
'put_object', service_response, expected_params)
try:
with self.stubber:
self.client.put_object(
Bucket='foo',
Key='bar.txt',
Metadata={
'MyMeta': 'Foo',
}
)
self.client.put_object(
Bucket='foo',
Key='bar.txt',
Metadata={
'MyMeta': 'Bar',
}
)
except StubAssertionError:
self.fail(
"stub.ANY failed to ignore nested parameter for validation.")
def test_ANY_repr(self):
self.assertEqual(repr(stub.ANY), '<ANY>')
def test_none_param(self):
service_response = {}
expected_params = {'Buck': None}
self.stubber.add_response(
'list_objects', service_response, expected_params)
self.stubber.activate()
# Throw an error for invalid parameters
with self.assertRaises(StubAssertionError):
self.client.list_objects(Buck='bar')
def test_many_expected_params(self):
service_response = {}
expected_params = {
'Bucket': 'mybucket',
'Prefix': 'myprefix',
'Delimiter': '/',
'EncodingType': 'url'
}
self.stubber.add_response(
'list_objects', service_response, expected_params)
try:
with self.stubber:
self.client.list_objects(**expected_params)
except StubAssertionError:
self.fail(
"Stubber inappropriately raised error for same parameters.")
def test_no_stub_for_presign_url(self):
try:
with self.stubber:
url = self.client.generate_presigned_url(
ClientMethod='get_object',
Params={
'Bucket': 'mybucket',
'Key': 'mykey'
}
)
self.assertEqual(
url, 'https://s3.amazonaws.com/mybucket/mykey')
except StubResponseError:
self.fail(
'Stubbed responses should not be required for generating '
'presigned requests'
)
def test_can_stub_with_presign_url_mixed_in(self):
desired_response = {}
expected_params = {
'Bucket': 'mybucket',
'Prefix': 'myprefix',
}
self.stubber.add_response(
'list_objects', desired_response, expected_params)
with self.stubber:
url = self.client.generate_presigned_url(
ClientMethod='get_object',
Params={
'Bucket': 'myotherbucket',
'Key': 'myotherkey'
}
)
self.assertEqual(
url, 'https://s3.amazonaws.com/myotherbucket/myotherkey')
actual_response = self.client.list_objects(**expected_params)
self.assertEqual(desired_response, actual_response)
self.stubber.assert_no_pending_responses()
class TestStubber(unittest.TestCase):
def setUp(self):
self.event_emitter = hooks.HierarchicalEmitter()
self.client = mock.Mock()
self.client.meta.events = self.event_emitter
self.client.meta.method_to_api_mapping.get.return_value = 'foo'
self.stubber = Stubber(self.client)
self.validate_parameters_mock = mock.Mock()
self.validate_parameters_patch = mock.patch(
'botocore.stub.validate_parameters', self.validate_parameters_mock)
self.validate_parameters_patch.start()
def tearDown(self):
self.validate_parameters_patch.stop()
def emit_get_response_event(self, model=None, request_dict=None,
signer=None, context=None):
if model is None:
model = mock.Mock()
model.name = 'foo'
handler, response = self.event_emitter.emit_until_response(
event_name='before-call.myservice.foo', model=model,
params=request_dict, request_signer=signer, context=context)
return response
def test_stubber_registers_events(self):
self.event_emitter = mock.Mock()
self.client.meta.events = self.event_emitter
self.stubber.activate()
# This just ensures that we register at the correct event
# and nothing more
self.event_emitter.register_first.assert_called_with(
'before-parameter-build.*.*', mock.ANY, unique_id=mock.ANY)
self.event_emitter.register.assert_called_with(
'before-call.*.*', mock.ANY, unique_id=mock.ANY)
def test_stubber_unregisters_events(self):
self.event_emitter = mock.Mock()
self.client.meta.events = self.event_emitter
self.stubber.activate()
self.stubber.deactivate()
self.event_emitter.unregister.assert_any_call(
'before-parameter-build.*.*', mock.ANY, unique_id=mock.ANY)
self.event_emitter.unregister.assert_any_call(
'before-call.*.*', mock.ANY, unique_id=mock.ANY)
def test_add_response(self):
response = {'foo': 'bar'}
self.stubber.add_response('foo', response)
with self.assertRaises(AssertionError):
self.stubber.assert_no_pending_responses()
def test_add_response_fails_when_missing_client_method(self):
del self.client.foo
with self.assertRaises(ValueError):
self.stubber.add_response('foo', {})
def test_validates_service_response(self):
self.stubber.add_response('foo', {})
self.assertTrue(self.validate_parameters_mock.called)
def test_validate_ignores_response_metadata(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
service_model = ServiceModel({
'documentation': '',
'operations': {
'foo': {
'name': 'foo',
'input': {'shape': 'StringShape'},
'output': {'shape': 'StringShape'}
}
},
'shapes': {
'StringShape': {'type': 'string'}
}
})
op_name = service_model.operation_names[0]
output_shape = service_model.operation_model(op_name).output_shape
self.client.meta.service_model = service_model
self.stubber.add_response('TestOperation', service_response)
self.validate_parameters_mock.assert_called_with(
{}, output_shape)
# Make sure service response hasn't been mutated
self.assertEqual(
service_response, {'ResponseMetadata': {'foo': 'bar'}})
def test_validates_on_empty_output_shape(self):
service_model = ServiceModel({
'documentation': '',
'operations': {
'foo': {
'name': 'foo'
}
}
})
self.client.meta.service_model = service_model
with self.assertRaises(ParamValidationError):
self.stubber.add_response('TestOperation', {'foo': 'bar'})
def test_get_response(self):
service_response = {'bar': 'baz'}
self.stubber.add_response('foo', service_response)
self.stubber.activate()
response = self.emit_get_response_event()
self.assertEqual(response[1], service_response)
self.assertEqual(response[0].status_code, 200)
def test_get_client_error_response(self):
error_code = "foo"
service_message = "bar"
self.stubber.add_client_error('foo', error_code, service_message)
self.stubber.activate()
response = self.emit_get_response_event()
self.assertEqual(response[1]['Error']['Message'], service_message)
self.assertEqual(response[1]['Error']['Code'], error_code)
def test_get_response_errors_with_no_stubs(self):
self.stubber.activate()
with self.assertRaises(StubResponseError):
self.emit_get_response_event()
def test_assert_no_responses_remaining(self):
self.stubber.add_response('foo', {})
with self.assertRaises(AssertionError):
self.stubber.assert_no_pending_responses()
class TestDeployer(unittest.TestCase):
def setUp(self):
client = botocore.session.get_session().create_client('cloudformation',
region_name="us-east-1")
self.stub_client = Stubber(client)
self.changeset_prefix = "some-changeset-prefix"
self.deployer = Deployer(client, self.changeset_prefix)
def test_has_stack_success(self):
stack_name = "stack_name"
expected_params = {
"StackName": stack_name
}
response = {
"Stacks": [
make_stack_obj(stack_name)
]
}
self.stub_client.add_response('describe_stacks', response,
expected_params)
with self.stub_client:
response = self.deployer.has_stack(stack_name)
self.assertTrue(response)
def test_has_stack_no_stack(self):
stack_name = "stack_name"
expected_params = {
"StackName": stack_name
}
# Response contains NO stack
no_stack_response = {
"Stacks": []
}
self.stub_client.add_response('describe_stacks', no_stack_response,
expected_params)
with self.stub_client:
response = self.deployer.has_stack(stack_name)
self.assertFalse(response)
# Response is a ClientError with a message that the stack does not exist
self.stub_client.add_client_error('describe_stacks', "ClientError",
"Stack with id {0} does not exist"
.format(stack_name))
with self.stub_client:
response = self.deployer.has_stack(stack_name)
self.assertFalse(response)
def test_has_stack_review_in_progress(self):
stack_name = "stack_name"
expected_params = {
"StackName": stack_name
}
# Response contains NO stack
review_in_progress_response = {
"Stacks": [make_stack_obj(stack_name, "REVIEW_IN_PROGRESS")]
}
self.stub_client.add_response('describe_stacks',
review_in_progress_response,
expected_params)
with self.stub_client:
response = self.deployer.has_stack(stack_name)
self.assertFalse(response)
def test_has_stack_exception(self):
self.stub_client.add_client_error('describe_stacks', "ValidationError",
"Service is bad")
with self.stub_client:
with self.assertRaises(botocore.exceptions.ClientError):
self.deployer.has_stack("stack_name")
def test_create_changeset_success(self):
stack_name = "stack_name"
template = "template"
parameters = [
{"ParameterKey": "Key1", "ParameterValue": "Value"},
{"ParameterKey": "Key2", "UsePreviousValue": True},
{"ParameterKey": "Key3", "UsePreviousValue": False},
]
# Parameters that Use Previous Value will be removed on stack creation
# to either force CloudFormation to use the Default value, or ask user to specify a parameter
filtered_parameters = [
{"ParameterKey": "Key1", "ParameterValue": "Value"},
{"ParameterKey": "Key3", "UsePreviousValue": False},
]
capabilities = ["capabilities"]
role_arn = "arn:aws:iam::1234567890:role"
notification_arns = ["arn:aws:sns:region:1234567890:notify"]
s3_uploader = None
tags = [{"Key":"key1", "Value": "val1"}]
# Case 1: Stack DOES NOT exist
self.deployer.has_stack = Mock()
self.deployer.has_stack.return_value = False
expected_params = {
"ChangeSetName": botocore.stub.ANY,
"StackName": stack_name,
"TemplateBody": template,
"ChangeSetType": "CREATE",
"Parameters": filtered_parameters,
"Capabilities": capabilities,
"Description": botocore.stub.ANY,
"RoleARN": role_arn,
"NotificationARNs": notification_arns,
"Tags": tags
}
response = {
"Id": "changeset ID"
}
self.stub_client.add_response("create_change_set", response,
expected_params)
with self.stub_client:
result = self.deployer.create_changeset(
stack_name, template, parameters, capabilities, role_arn,
notification_arns, s3_uploader, tags)
self.assertEquals(response["Id"], result.changeset_id)
self.assertEquals("CREATE", result.changeset_type)
# Case 2: Stack exists. We are updating it
self.deployer.has_stack.return_value = True
self.stub_client.add_response("get_template_summary",
{"Parameters": [{"ParameterKey": parameter["ParameterKey"]}
for parameter in parameters]},
{"StackName": stack_name})
expected_params["ChangeSetType"] = "UPDATE"
expected_params["Parameters"] = parameters
self.stub_client.add_response("create_change_set", response,
expected_params)
# template has new parameter but should not be included in
# expected_params as no previous value
parameters = list(parameters) + \
[{"ParameterKey": "New", "UsePreviousValue": True}]
with self.stub_client:
result = self.deployer.create_changeset(
stack_name, template, parameters, capabilities, role_arn,
notification_arns, s3_uploader, tags)
self.assertEquals(response["Id"], result.changeset_id)
self.assertEquals("UPDATE", result.changeset_type)
def test_create_changeset_success_s3_bucket(self):
stack_name = "stack_name"
template = "template"
template_url = "https://s3.amazonaws.com/bucket/file"
parameters = [
{"ParameterKey": "Key1", "ParameterValue": "Value"},
{"ParameterKey": "Key2", "UsePreviousValue": True},
{"ParameterKey": "Key3", "UsePreviousValue": False},
]
# Parameters that Use Previous Value will be removed on stack creation
# to either force CloudFormation to use the Default value, or ask user to specify a parameter
filtered_parameters = [
{"ParameterKey": "Key1", "ParameterValue": "Value"},
{"ParameterKey": "Key3", "UsePreviousValue": False},
]
capabilities = ["capabilities"]
role_arn = "arn:aws:iam::1234567890:role"
notification_arns = ["arn:aws:sns:region:1234567890:notify"]
s3_uploader = Mock()
def to_path_style_s3_url(some_string, Version=None):
return "https://s3.amazonaws.com/bucket/file"
s3_uploader.to_path_style_s3_url = to_path_style_s3_url
def upload_with_dedup(filename,extension):
return "s3://bucket/file"
s3_uploader.upload_with_dedup = upload_with_dedup
# Case 1: Stack DOES NOT exist
self.deployer.has_stack = Mock()
self.deployer.has_stack.return_value = False
expected_params = {
"ChangeSetName": botocore.stub.ANY,
"StackName": stack_name,
"TemplateURL": template_url,
"ChangeSetType": "CREATE",
"Parameters": filtered_parameters,
"Capabilities": capabilities,
"Description": botocore.stub.ANY,
"RoleARN": role_arn,
"Tags": [],
"NotificationARNs": notification_arns
}
response = {
"Id": "changeset ID"
}
self.stub_client.add_response("create_change_set", response,
expected_params)
with self.stub_client:
result = self.deployer.create_changeset(
stack_name, template, parameters, capabilities, role_arn,
notification_arns, s3_uploader, [])
self.assertEquals(response["Id"], result.changeset_id)
self.assertEquals("CREATE", result.changeset_type)
# Case 2: Stack exists. We are updating it
self.deployer.has_stack.return_value = True
self.stub_client.add_response("get_template_summary",
{"Parameters": [{"ParameterKey": parameter["ParameterKey"]}
for parameter in parameters]},
{"StackName": stack_name})
expected_params["ChangeSetType"] = "UPDATE"
expected_params["Parameters"] = parameters
# template has new parameter but should not be included in
# expected_params as no previous value
parameters = list(parameters) + \
[{"ParameterKey": "New", "UsePreviousValue": True}]
self.stub_client.add_response("create_change_set", response,
expected_params)
with self.stub_client:
result = self.deployer.create_changeset(
stack_name, template, parameters, capabilities, role_arn,
notification_arns, s3_uploader, [])
self.assertEquals(response["Id"], result.changeset_id)
self.assertEquals("UPDATE", result.changeset_type)
def test_create_changeset_exception(self):
stack_name = "stack_name"
template = "template"
parameters = [{"ParameterKey": "Key1", "ParameterValue": "Value",
"UsePreviousValue": True}]
capabilities = ["capabilities"]
role_arn = "arn:aws:iam::1234567890:role"
notification_arns = ["arn:aws:sns:region:1234567890:notify"]
s3_uploader = None
tags = [{"Key":"key1", "Value": "val1"}]
self.deployer.has_stack = Mock()
self.deployer.has_stack.return_value = False
self.stub_client.add_client_error(
'create_change_set', "Somethign is wrong", "Service is bad")
with self.stub_client:
with self.assertRaises(botocore.exceptions.ClientError):
self.deployer.create_changeset(stack_name, template, parameters,
capabilities, role_arn, notification_arns, None, tags)
def test_execute_changeset(self):
stack_name = "stack_name"
changeset_id = "changeset_id"
expected_params = {
"ChangeSetName": changeset_id,
"StackName": stack_name
}
self.stub_client.add_response("execute_change_set", {}, expected_params)
with self.stub_client:
self.deployer.execute_changeset(changeset_id, stack_name)
def test_execute_changeset_exception(self):
stack_name = "stack_name"
changeset_id = "changeset_id"
self.stub_client.add_client_error(
'execute_change_set', "Somethign is wrong", "Service is bad")
with self.stub_client:
with self.assertRaises(botocore.exceptions.ClientError):
self.deployer.execute_changeset(changeset_id, stack_name)
def test_create_and_wait_for_changeset_successful(self):
stack_name = "stack_name"
template = "template"
parameters = [{"ParameterKey": "Key1", "ParameterValue": "Value",
"UsePreviousValue": True}]
capabilities = ["capabilities"]
changeset_id = "changeset id"
changeset_type = "changeset type"
role_arn = "arn:aws:iam::1234567890:role"
notification_arns = ["arn:aws:sns:region:1234567890:notify"]
s3_uploader = None
tags = [{"Key":"key1", "Value": "val1"}]
self.deployer.create_changeset = Mock()
self.deployer.create_changeset.return_value = ChangeSetResult(changeset_id, changeset_type)
self.deployer.wait_for_changeset = Mock()
result = self.deployer.create_and_wait_for_changeset(
stack_name, template, parameters, capabilities, role_arn,
notification_arns, s3_uploader, tags)
self.assertEquals(result.changeset_id, changeset_id)
self.assertEquals(result.changeset_type, changeset_type)
def test_create_and_wait_for_changeset_error_waiting_for_changeset(self):
stack_name = "stack_name"
template = "template"
parameters = [{"ParameterKey": "Key1", "ParameterValue": "Value",
"UsePreviousValue": True}]
capabilities = ["capabilities"]
changeset_id = "changeset id"
changeset_type = "changeset type"
role_arn = "arn:aws:iam::1234567890:role"
notification_arns = ["arn:aws:sns:region:1234567890:notify"]
s3_uploader = None
tags = [{"Key":"key1", "Value": "val1"}]
self.deployer.create_changeset = Mock()
self.deployer.create_changeset.return_value = ChangeSetResult(changeset_id, changeset_type)
self.deployer.wait_for_changeset = Mock()
self.deployer.wait_for_changeset.side_effect = RuntimeError
with self.assertRaises(RuntimeError):
result = self.deployer.create_and_wait_for_changeset(
stack_name, template, parameters, capabilities, role_arn,
notification_arns, s3_uploader, tags)
def test_wait_for_changeset_no_changes(self):
stack_name = "stack_name"
changeset_id = "changeset-id"
mock_client = Mock()
mock_deployer = Deployer(mock_client)
mock_waiter = Mock()
mock_client.get_waiter.return_value = mock_waiter
response = {
"Status": "FAILED",
"StatusReason": "The submitted information didn't contain changes."
}
waiter_error = botocore.exceptions.WaiterError(name="name",
reason="reason",
last_response=response)
mock_waiter.wait.side_effect = waiter_error
with self.assertRaises(exceptions.ChangeEmptyError):
mock_deployer.wait_for_changeset(changeset_id, stack_name)
waiter_config = {'Delay': 5}
mock_waiter.wait.assert_called_once_with(ChangeSetName=changeset_id,
StackName=stack_name,
WaiterConfig=waiter_config)
mock_client.get_waiter.assert_called_once_with(
"change_set_create_complete")
def test_wait_for_changeset_no_changes_with_another_error_msg(self):
stack_name = "stack_name"
changeset_id = "changeset-id"
mock_client = Mock()
mock_deployer = Deployer(mock_client)
mock_waiter = Mock()
mock_client.get_waiter.return_value = mock_waiter
response = {
"Status": "FAILED",
"StatusReason": "No updates are to be performed"
}
waiter_error = botocore.exceptions.WaiterError(name="name",
reason="reason",
last_response=response)
mock_waiter.wait.side_effect = waiter_error
with self.assertRaises(exceptions.ChangeEmptyError):
mock_deployer.wait_for_changeset(changeset_id, stack_name)
waiter_config = {'Delay': 5}
mock_waiter.wait.assert_called_once_with(ChangeSetName=changeset_id,
StackName=stack_name,
WaiterConfig=waiter_config)
mock_client.get_waiter.assert_called_once_with(
"change_set_create_complete")
def test_wait_for_changeset_failed_to_create_changeset(self):
stack_name = "stack_name"
changeset_id = "changeset-id"
mock_client = Mock()
mock_deployer = Deployer(mock_client)
mock_waiter = Mock()
mock_client.get_waiter.return_value = mock_waiter
response = {
"Status": "FAILED",
"StatusReason": "some reason"
}
waiter_error = botocore.exceptions.WaiterError(name="name",
reason="reason",
last_response=response)
mock_waiter.wait.side_effect = waiter_error
with self.assertRaises(RuntimeError):
mock_deployer.wait_for_changeset(changeset_id, stack_name)
waiter_config = {'Delay': 5}
mock_waiter.wait.assert_called_once_with(ChangeSetName=changeset_id,
StackName=stack_name,
WaiterConfig=waiter_config)
mock_client.get_waiter.assert_called_once_with(
"change_set_create_complete")
def test_wait_for_execute_no_changes(self):
stack_name = "stack_name"
changeset_type = "CREATE"
mock_client = Mock()
mock_deployer = Deployer(mock_client)
mock_waiter = Mock()
mock_client.get_waiter.return_value = mock_waiter
waiter_error = botocore.exceptions.WaiterError(name="name",
reason="reason",
last_response={})
mock_waiter.wait.side_effect = waiter_error
with self.assertRaises(exceptions.DeployFailedError):
mock_deployer.wait_for_execute(stack_name, changeset_type)
waiter_config = {
'Delay': 5,
'MaxAttempts': 720,
}
mock_waiter.wait.assert_called_once_with(StackName=stack_name,
WaiterConfig=waiter_config)
mock_client.get_waiter.assert_called_once_with(
"stack_create_complete")
