class TestRoute53Pagination(unittest.TestCase):
def setUp(self):
self.session = botocore.session.get_session()
self.client = self.session.create_client('route53', 'us-west-2')
self.stubber = Stubber(self.client)
# response has required fields
self.response = {
'HostedZones': [],
'Marker': '',
'IsTruncated': True,
'MaxItems': '1'
}
self.operation_name = 'list_hosted_zones'
def test_paginate_with_max_items_int(self):
# Route53 has a string type for MaxItems. We need to ensure that this
# still works with integers as the cli auto converts the page size
# argument to an integer.
self.stubber.add_response(self.operation_name, self.response)
paginator = self.client.get_paginator('list_hosted_zones')
with self.stubber:
config={'PageSize': 1}
results = list(paginator.paginate(PaginationConfig=config))
self.assertTrue(len(results) >= 0)
def test_paginate_with_max_items_str(self):
# Route53 has a string type for MaxItems. We need to ensure that this
# still works with strings as that's the expected type for this key.
self.stubber.add_response(self.operation_name, self.response)
paginator = self.client.get_paginator('list_hosted_zones')
with self.stubber:
config={'PageSize': '1'}
results = list(paginator.paginate(PaginationConfig=config))
self.assertTrue(len(results) >= 0)
def test_cluster_info(cluster_provisioner):
cluster_id = 'foo-bar-spam-egs'
stubber = Stubber(cluster_provisioner.emr)
response = {
'Cluster': {
'MasterPublicDnsName': '1.2.3.4',
'Status': {
'State': 'RUNNING',
'StateChangeReason': {
'Code': 'ALL_STEPS_COMPLETED',
'Message': 'All steps completed.',
},
'Timeline': {
'CreationDateTime': datetime(2015, 1, 1),
'ReadyDateTime': datetime(2015, 1, 2),
'EndDateTime': datetime(2015, 1, 3),
}
},
},
}
expected_params = {'ClusterId': cluster_id}
stubber.add_response('describe_cluster', response, expected_params)
with stubber:
info = cluster_provisioner.info(cluster_id)
assert info == {
'creation_datetime': datetime(2015, 1, 1),
'ready_datetime': datetime(2015, 1, 2),
'end_datetime': datetime(2015, 1, 3),
'state_change_reason_code': 'ALL_STEPS_COMPLETED',
'state_change_reason_message': 'All steps completed.',
'state': 'RUNNING',
'public_dns': '1.2.3.4',
}
class TestIdempotencyToken(unittest.TestCase):
def setUp(self):
self.function_name = "purchase_scheduled_instances"
self.region = "us-west-2"
self.session = botocore.session.get_session()
self.client = self.session.create_client("ec2", self.region)
self.stubber = Stubber(self.client)
self.service_response = {}
self.params_seen = []
# Record all the parameters that get seen
self.client.meta.events.register_first("before-call.*.*", self.collect_params, unique_id="TestIdempotencyToken")
def collect_params(self, model, params, *args, **kwargs):
self.params_seen.extend(params["body"].keys())
def test_provided_idempotency_token(self):
expected_params = {"PurchaseRequests": [{"PurchaseToken": "foo", "InstanceCount": 123}], "ClientToken": ANY}
self.stubber.add_response(self.function_name, self.service_response, expected_params)
with self.stubber:
self.client.purchase_scheduled_instances(
PurchaseRequests=[{"PurchaseToken": "foo", "InstanceCount": 123}], ClientToken="foobar"
)
self.assertIn("ClientToken", self.params_seen)
def test_insert_idempotency_token(self):
expected_params = {"PurchaseRequests": [{"PurchaseToken": "foo", "InstanceCount": 123}]}
self.stubber.add_response(self.function_name, self.service_response, expected_params)
with self.stubber:
self.client.purchase_scheduled_instances(PurchaseRequests=[{"PurchaseToken": "foo", "InstanceCount": 123}])
self.assertIn("ClientToken", self.params_seen)
def test_ensure_cfn_bucket_doesnt_exist_us_west(self):
session = get_session("us-west-1")
provider = Provider(session)
action = BaseAction(
context=mock_context("mynamespace"),
provider_builder=MockProviderBuilder(provider, region="us-west-1")
)
stubber = Stubber(action.s3_conn)
stubber.add_client_error(
"head_bucket",
service_error_code="NoSuchBucket",
service_message="Not Found",
http_status_code=404,
)
stubber.add_response(
"create_bucket",
service_response={},
expected_params={
"Bucket": ANY,
"CreateBucketConfiguration": {
"LocationConstraint": "us-west-1",
}
}
)
with stubber:
action.ensure_cfn_bucket()
def test_mutating_filters(self):
stubber = Stubber(self.service_resource.meta.client)
instance_filters = [
{'Name': 'instance-state-name', 'Values': ['running']}
]
running_instances = self.service_resource.instances.filter(
Filters=instance_filters
)
# This should not impact the already-created filter.
instance_filters.append(
{'Name': 'instance-type', 'Values': ['c4.large']}
)
stubber.add_response(
method='describe_instances',
service_response={
'Reservations': []
},
expected_params={
'Filters': [{
'Name': 'instance-state-name',
'Values': ['running']
}]
}
)
with stubber:
list(running_instances)
stubber.assert_no_pending_responses()
def boto_volume_for_test(test, cluster_id):
"""
Create an in-memory boto3 Volume, avoiding any AWS API calls.
"""
# Create a session directly rather than allow lazy loading of a default
# session.
region_name = u"some-test-region-1"
s = Boto3Session(
botocore_session=botocore_get_session(),
region_name=region_name,
)
ec2 = s.resource("ec2", region_name=region_name)
stubber = Stubber(ec2.meta.client)
# From this point, any attempt to interact with AWS API should fail with
# botocore.exceptions.StubResponseError
stubber.activate()
volume_id = u"vol-{}".format(random_name(test))
v = ec2.Volume(id=volume_id)
tags = []
if cluster_id is not None:
tags.append(
dict(
Key=CLUSTER_ID_LABEL,
Value=cluster_id,
),
)
# Pre-populate the metadata to prevent any attempt to load the metadata by
# API calls.
v.meta.data = dict(
Tags=tags
)
return v
def create_client_sts_stub(service, *args, **kwargs):
client = _original_create_client(service, *args, **kwargs)
stub = Stubber(client)
response = self.create_assume_role_response(expected_creds)
self.actual_client_region = client.meta.region_name
stub.add_response('assume_role', response)
stub.activate()
return client
def test_stop_cluster(cluster_provisioner):
stubber = Stubber(cluster_provisioner.emr)
response = {}
expected_params = {
'JobFlowIds': ['12345'],
}
stubber.add_response('terminate_job_flows', response, expected_params)
with stubber:
cluster_provisioner.stop(jobflow_id='12345')
class TestLogsCommandContext_get_resource_id_from_stack(TestCase):
def setUp(self):
self.real_client = botocore.session.get_session().create_client('cloudformation', region_name="us-east-1")
self.cfn_client_stubber = Stubber(self.real_client)
self.logical_id = "name"
self.stack_name = "stackname"
self.physical_id = "myid"
def test_must_get_from_cfn(self):
expected_params = {
"StackName": self.stack_name,
"LogicalResourceId": self.logical_id
}
mock_response = {
"StackResourceDetail": {
"PhysicalResourceId": self.physical_id,
"LogicalResourceId": self.logical_id,
"ResourceType": "AWS::Lambda::Function",
"ResourceStatus": "UPDATE_COMPLETE",
"LastUpdatedTimestamp": "2017-07-28T23:34:13.435Z"
}
}
self.cfn_client_stubber.add_response("describe_stack_resource", mock_response, expected_params)
with self.cfn_client_stubber:
result = LogsCommandContext._get_resource_id_from_stack(self.real_client,
self.stack_name,
self.logical_id)
self.assertEquals(result, self.physical_id)
def test_must_handle_resource_not_found(self):
errmsg = "Something went wrong"
errcode = "SomeException"
self.cfn_client_stubber.add_client_error("describe_stack_resource",
service_error_code=errcode,
service_message=errmsg)
expected_error_msg = "An error occurred ({}) when calling the DescribeStackResource operation: {}".format(
errcode, errmsg)
with self.cfn_client_stubber:
with self.assertRaises(UserException) as context:
LogsCommandContext._get_resource_id_from_stack(self.real_client,
self.stack_name,
self.logical_id)
self.assertEquals(expected_error_msg, str(context.exception))
def test_delete_tags(self):
stubber = Stubber(self.instance_resource.meta.client)
stubber.add_response('delete_tags', {})
stubber.activate()
response = self.instance_resource.delete_tags(Tags=[{'Key': 'foo'}])
stubber.assert_no_pending_responses()
self.assertEqual(response, {})
stubber.deactivate()
class TestIdempotencyToken(unittest.TestCase):
def setUp(self):
self.function_name = 'purchase_scheduled_instances'
self.region = 'us-west-2'
self.session = botocore.session.get_session()
self.client = self.session.create_client(
'ec2', self.region)
self.stubber = Stubber(self.client)
self.service_response = {}
self.params_seen = []
# Record all the parameters that get seen
self.client.meta.events.register_first(
'before-call.*.*',
self.collect_params,
unique_id='TestIdempotencyToken')
def collect_params(self, model, params, *args, **kwargs):
self.params_seen.extend(params['body'].keys())
def test_provided_idempotency_token(self):
expected_params = {
'PurchaseRequests': [
{'PurchaseToken': 'foo',
'InstanceCount': 123}],
'ClientToken': ANY
}
self.stubber.add_response(
self.function_name, self.service_response, expected_params)
with self.stubber:
self.client.purchase_scheduled_instances(
PurchaseRequests=[{'PurchaseToken': 'foo',
'InstanceCount': 123}],
ClientToken='foobar')
self.assertIn('ClientToken', self.params_seen)
def test_insert_idempotency_token(self):
expected_params = {
'PurchaseRequests': [
{'PurchaseToken': 'foo',
'InstanceCount': 123}],
}
self.stubber.add_response(
self.function_name, self.service_response, expected_params)
with self.stubber:
self.client.purchase_scheduled_instances(
PurchaseRequests=[{'PurchaseToken': 'foo',
'InstanceCount': 123}])
self.assertIn('ClientToken', self.params_seen)
def test_spark_job_remove(spark_job_provisioner):
key = 's3://test/test-notebook.ipynb'
stubber = Stubber(spark_job_provisioner.s3)
response = {'DeleteMarker': False}
expected_params = {
'Bucket': settings.AWS_CONFIG['CODE_BUCKET'],
'Key': key,
}
stubber.add_response('delete_object', response, expected_params)
with stubber:
spark_job_provisioner.remove(key)
def test_create_cluster_valid_parameters():
"""Test that the parameters passed down to run_job_flow are valid"""
stubber = Stubber(emr)
response = {'JobFlowId': 'job-flow-id'}
stubber.add_response('run_job_flow', response)
emr_release = settings.AWS_CONFIG['EMR_RELEASES'][0]
params = ['*****@*****.**', 'cluster', 3, 'public-key', emr_release]
with stubber:
job_flow_id = provisioning.cluster_start(*params)
assert job_flow_id == response['JobFlowId']
def test_operation_without_output(self):
table = self.resource.Table('mytable')
stubber = Stubber(table.meta.client)
stubber.add_response('tag_resource', {})
arn = 'arn:aws:dynamodb:us-west-2:123456789:table/mytable'
with stubber:
table.meta.client.tag_resource(
ResourceArn=arn,
Tags=[{'Key': 'project', 'Value': 'val'}]
)
stubber.assert_no_pending_responses()
def setUp(self):
self.stubber = Stubber(self.client)
self.get_parameters_response = {
'Parameters': [
{
'Name': 'ssmkey',
'Type': 'String',
'Value': 'ssmvalue'
}
],
'InvalidParameters': [
'invalidssmparam'
]
}
self.invalid_get_parameters_response = {
'InvalidParameters': [
'ssmkey'
]
}
self.expected_params = {
'Names': ['ssmkey'],
'WithDecryption': True
}
self.ssmkey = "ssmkey"
self.ssmvalue = "ssmvalue"
def test_spark_job_get(spark_job_provisioner):
key = 's3://test/test-notebook.ipynb'
stubber = Stubber(spark_job_provisioner.s3)
response = {
'Body': 'content',
}
expected_params = {
'Bucket': settings.AWS_CONFIG['CODE_BUCKET'],
'Key': key,
}
stubber.add_response('get_object', response, expected_params)
with stubber:
result = spark_job_provisioner.get(key)
assert result == response
def setUp(self):
super(TestMturk, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
'mturk', self.region)
self.stubber = Stubber(self.client)
self.stubber.activate()
def setUp(self):
client = botocore.session.get_session().create_client('cloudformation',
region_name="us-east-1")
self.stub_client = Stubber(client)
self.changeset_prefix = "some-changeset-prefix"
self.deployer = Deployer(client, self.changeset_prefix)
def test_table_scan_can_be_stubbed_with_expressions(self):
table = self.resource.Table('mytable')
filter_expr = Attr('myattr').eq('foo') & (
Attr('myattr2').lte('buzz') | Attr('myattr2').gte('fizz')
)
stubber = Stubber(table.meta.client)
stubber.add_response('scan', dict(Items=list()), expected_params=dict(
TableName='mytable',
FilterExpression=filter_expr
))
with stubber:
response = table.scan(FilterExpression=filter_expr)
self.assertEqual(list(), response['Items'])
stubber.assert_no_pending_responses()
def setUp(self):
self.stubber = Stubber(self.client)
self.get_parameters_response = {'Item': {'TestMap': {'M': {
'String1': {'S': 'StringVal1'},
'List1': {'L': [
{'S': 'ListVal1'},
{'S': 'ListVal2'}]},
'Number1': {'N': '12345'}, }}}}
def setUp(self):
self.real_client = botocore.session.get_session().create_client('cloudformation', region_name="us-east-1")
self.cfn_client_stubber = Stubber(self.real_client)
self.logical_id = "name"
self.stack_name = "stackname"
self.physical_id = "myid"
def setUp(self):
super(TestSagemaker, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
'sagemaker', self.region)
self.stubber = Stubber(self.client)
self.stubber.activate()
self.hook_calls = []
def test_ensure_cfn_forbidden(self):
session = get_session("us-west-1")
provider = Provider(session)
action = BaseAction(
context=mock_context("mynamespace"),
provider_builder=MockProviderBuilder(provider)
)
stubber = Stubber(action.s3_conn)
stubber.add_client_error(
"head_bucket",
service_error_code="AccessDenied",
service_message="Forbidden",
http_status_code=403,
)
with stubber:
with self.assertRaises(botocore.exceptions.ClientError):
action.ensure_cfn_bucket()
def test_ensure_cfn_bucket_exists(self):
session = get_session("us-east-1")
provider = Provider(session)
action = BaseAction(
context=mock_context("mynamespace"),
provider_builder=MockProviderBuilder(provider)
)
stubber = Stubber(action.s3_conn)
stubber.add_response(
"head_bucket",
service_response={},
expected_params={
"Bucket": ANY,
}
)
with stubber:
action.ensure_cfn_bucket()
def setUp(self):
self.session = botocore.session.get_session()
self.region = 'us-west-2'
self.client = self.session.create_client(
's3', self.region, aws_access_key_id='foo',
aws_secret_access_key='bar')
self.stubber = Stubber(self.client)
self.stubber.activate()
def create_session(self, profile=None):
session = Session(profile=profile)
# We have to set bogus credentials here or otherwise we'll trigger
# an early credential chain resolution.
sts = session.create_client(
'sts',
aws_access_key_id='spam',
aws_secret_access_key='eggs',
)
stubber = Stubber(sts)
stubber.activate()
assume_role_provider = AssumeRoleProvider(
load_config=lambda: session.full_config,
client_creator=lambda *args, **kwargs: sts,
cache={},
profile_name=profile,
credential_sourcer=CanonicalNameCredentialSourcer([
self.env_provider, self.container_provider,
self.metadata_provider
])
)
component_name = 'credential_provider'
resolver = session.get_component(component_name)
available_methods = [p.METHOD for p in resolver.providers]
replacements = {
'env': self.env_provider,
'iam-role': self.metadata_provider,
'container-role': self.container_provider,
'assume-role': assume_role_provider
}
for name, provider in replacements.items():
try:
index = available_methods.index(name)
except ValueError:
# The provider isn't in the session
continue
resolver.providers[index] = provider
session.register_component(
'credential_provider', resolver
)
return session, stubber
def setUp(self):
super(TestAutoscalingPagination, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
'autoscaling', self.region, aws_secret_access_key='foo',
aws_access_key_id='bar', aws_session_token='baz'
)
self.stubber = Stubber(self.client)
self.stubber.activate()
def setUp(self):
session = botocore.session.get_session()
config = botocore.config.Config(
signature_version=botocore.UNSIGNED,
s3={'addressing_style': 'path'}
)
self.client = session.create_client(
's3', region_name='us-east-1', config=config)
self.stubber = Stubber(self.client)
def setUp(self):
self.event_emitter = hooks.HierarchicalEmitter()
self.client = mock.Mock()
self.client.meta.events = self.event_emitter
self.client.meta.method_to_api_mapping.get.return_value = 'foo'
self.stubber = Stubber(self.client)
self.validate_parameters_mock = mock.Mock()
self.validate_parameters_patch = mock.patch(
'botocore.stub.validate_parameters', self.validate_parameters_mock)
self.validate_parameters_patch.start()
def test_create_cluster_valid_parameters(cluster_provisioner):
"""Test that the parameters passed down to run_job_flow are valid"""
stubber = Stubber(cluster_provisioner.emr)
response = {'JobFlowId': 'job-flow-id'}
stubber.add_response('run_job_flow', response)
emr_release = '5.0.0'
with stubber:
jobflow_id = cluster_provisioner.start(
user_username='******',
user_email='*****@*****.**',
identifier='cluster',
emr_release=emr_release,
size=3,
public_key='public-key',
)
assert jobflow_id == response['JobFlowId']
def test_handle_ses_sending_quota_warning(monitor, target_datetime):
ses_stubber = Stubber(monitor.ses_service.client)
ses_stubber.add_response('get_send_quota', {
'Max24HourSend': 10.0,
'MaxSendRate': 523.0,
'SentLast24Hours': 8.0
}, {})
ses_stubber.activate()
result = monitor.handle_ses_sending_quota(target_datetime=target_datetime)
assert len(result['pager_duty']) == 1
assert result['pager_duty'][0] == {
'dedup_key':
'undefined-None-undefined-ses-account-monitor/ses_account_sending_quota',
'event_action': 'resolve',
'routing_key': None
}
assert len(result['slack']) == 1
assert result['slack'][0] == {
'attachments': [{
'color':
'warning',
'fallback':
'SES account sending rate has breached WARNING threshold.',
'fields': [{
'short':
True,
'title':
'Service',
'value':
'<https://None.console.aws.amazon.com/ses/home?region=None#dashboard:|SES Account Sending>'
}, {
'short': True,
'title': 'Account',
'value': 'undefined'
}, {
'short': True,
'title': 'Region',
'value': None
}, {
'short': True,
'title': 'Environment',
'value': 'undefined'
}, {
'short': True,
'title': 'Status',
'value': 'WARNING'
}, {
'title': 'Time',
'value': '2018-01-01T00:00:00+00:00'
}, {
'short': True,
'title': 'Utilization',
'value': '80.00%'
}, {
'short': True,
'title': 'Threshold',
'value': '80.00%'
}, {
'short': True,
'title': 'Volume',
'value': 8.0
}, {
'short': True,
'title': 'Max Volume',
'value': 10.0
}, {
'short':
False,
'title':
'Message',
'value':
'SES account sending rate has breached the WARNING threshold.'
}],
'footer':
'undefined-None-undefined-ses-account-monitor',
'footer_icon':
'https://platform.slack-edge.com/img/default_application_icon.png',
'ts':
1514764800
}],
'icon_emoji':
None,
'username':
'******'
}
class TestStubber(unittest.TestCase):
def setUp(self):
session = botocore.session.get_session()
config = botocore.config.Config(signature_version=botocore.UNSIGNED)
self.client = session.create_client('s3', config=config)
self.stubber = Stubber(self.client)
def test_stubber_returns_response(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
self.stubber.add_response('list_objects', service_response)
self.stubber.activate()
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_context_manager_returns_response(self):
service_response = {'ResponseMetadata': {'foo': 'bar'}}
self.stubber.add_response('list_objects', service_response)
with self.stubber:
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_activated_stubber_errors_with_no_registered_stubs(self):
self.stubber.activate()
# Params one per line for readability.
with self.assertRaisesRegexp(StubResponseError,
"'Bucket': 'asdfasdfasdfasdf',\n"):
self.client.list_objects(Bucket='asdfasdfasdfasdf',
Delimiter='asdfasdfasdfasdf',
Prefix='asdfasdfasdfasdf',
EncodingType='url')
def test_stubber_errors_when_stubs_are_used_up(self):
self.stubber.add_response('list_objects', {})
self.stubber.activate()
self.client.list_objects(Bucket='foo')
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
def test_client_error_response(self):
error_code = "AccessDenied"
error_message = "Access Denied"
self.stubber.add_client_error('list_objects', error_code,
error_message)
self.stubber.activate()
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
def test_can_add_expected_params_to_client_error(self):
self.stubber.add_client_error('list_objects',
'Error',
'error',
expected_params={'Bucket': 'foo'})
self.stubber.activate()
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
def test_can_expected_param_fails_in_client_error(self):
self.stubber.add_client_error('list_objects',
'Error',
'error',
expected_params={'Bucket': 'foo'})
self.stubber.activate()
# We expect an AssertionError instead of a ClientError
# because we're calling the operation with the wrong
# param value.
with self.assertRaises(AssertionError):
self.client.list_objects(Bucket='wrong-argument-value')
def test_expected_params_success(self):
service_response = {}
expected_params = {'Bucket': 'foo'}
self.stubber.add_response('list_objects', service_response,
expected_params)
self.stubber.activate()
# This should be called successfully with no errors being thrown
# for mismatching expected params.
response = self.client.list_objects(Bucket='foo')
self.assertEqual(response, service_response)
def test_expected_params_fail(self):
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response('list_objects', service_response,
expected_params)
self.stubber.activate()
# This should call should raise an for mismatching expected params.
with self.assertRaisesRegexp(StubResponseError,
"{'Bucket': 'bar'},\n"):
self.client.list_objects(Bucket='foo')
def test_expected_params_mixed_with_errors_responses(self):
# Add an error response
error_code = "AccessDenied"
error_message = "Access Denied"
self.stubber.add_client_error('list_objects', error_code,
error_message)
# Add a response with incorrect expected params
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response('list_objects', service_response,
expected_params)
self.stubber.activate()
# The first call should throw and error as expected.
with self.assertRaises(ClientError):
self.client.list_objects(Bucket='foo')
# The second call should throw an error for unexpected parameters
with self.assertRaisesRegexp(StubResponseError, 'Expected parameters'):
self.client.list_objects(Bucket='foo')
def test_can_continue_to_call_after_expected_params_fail(self):
service_response = {}
expected_params = {'Bucket': 'bar'}
self.stubber.add_response('list_objects', service_response,
expected_params)
self.stubber.activate()
# Throw an error for unexpected parameters
with self.assertRaises(StubResponseError):
self.client.list_objects(Bucket='foo')
# The stubber should still have the responses queued up
# even though the original parameters did not match the expected ones.
self.client.list_objects(Bucket='bar')
self.stubber.assert_no_pending_responses()
def test_still_relies_on_param_validation_with_expected_params(self):
service_response = {}
expected_params = {'Buck': 'bar'}
self.stubber.add_response('list_objects', service_response,
expected_params)
self.stubber.activate()
# Throw an error for invalid parameters
with self.assertRaises(ParamValidationError):
self.client.list_objects(Buck='bar')
def test_any_ignores_param_for_validation(self):
service_response = {}
expected_params = {'Bucket': stub.ANY}
self.stubber.add_response('list_objects', service_response,
expected_params)
self.stubber.add_response('list_objects', service_response,
expected_params)
try:
with self.stubber:
self.client.list_objects(Bucket='foo')
self.client.list_objects(Bucket='bar')
except StubAssertionError:
self.fail("stub.ANY failed to ignore parameter for validation.")
def test_mixed_any_and_concrete_params(self):
service_response = {}
expected_params = {'Bucket': stub.ANY, 'Key': 'foo.txt'}
self.stubber.add_response('head_object', service_response,
expected_params)
self.stubber.add_response('head_object', service_response,
expected_params)
try:
with self.stubber:
self.client.head_object(Bucket='foo', Key='foo.txt')
self.client.head_object(Bucket='bar', Key='foo.txt')
except StubAssertionError:
self.fail("stub.ANY failed to ignore parameter for validation.")
def test_none_param(self):
service_response = {}
expected_params = {'Buck': None}
self.stubber.add_response('list_objects', service_response,
expected_params)
self.stubber.activate()
# Throw an error for invalid parameters
with self.assertRaises(StubAssertionError):
self.client.list_objects(Buck='bar')
def test_many_expected_params(self):
service_response = {}
expected_params = {
'Bucket': 'mybucket',
'Prefix': 'myprefix',
'Delimiter': '/',
'EncodingType': 'url'
}
self.stubber.add_response('list_objects', service_response,
expected_params)
try:
with self.stubber:
self.client.list_objects(**expected_params)
except StubAssertionError:
self.fail(
"Stubber inappropriately raised error for same parameters.")
def test_handle_ses_reputation_critical(monitor, end_datetime,
metric_data_results_response_critical,
metric_data_results_params):
cloudwatch_stubber = Stubber(monitor.cloudwatch_service.client)
cloudwatch_stubber.add_response('get_metric_data',
metric_data_results_response_critical,
metric_data_results_params)
cloudwatch_stubber.activate()
result = monitor.handle_ses_reputation(target_datetime=end_datetime)
assert len(result['pager_duty']) == 1
assert result['pager_duty'][0] == {
'client': 'AWS Console',
'client_url':
'https://None.console.aws.amazon.com/ses/home?region=None#reputation-dashboard:',
'dedup_key':
'undefined-None-undefined-ses-account-monitor/ses_account_reputation',
'event_action': 'trigger',
'payload': {
'class': 'ses_account_reputation',
'component': 'ses',
'custom_details': {
'action': 'alert',
'action_message':
'SES account is in danger of being suspended.',
'aws_account_name': 'undefined',
'aws_environment': 'undefined',
'aws_region': None,
'bounce_rate': '5.00%',
'bounce_rate_threshold': '5.00%',
'bounce_rate_timestamp': '2018-06-17T02:11:25.787402+00:00',
'complaint_rate': '99.00%',
'complaint_rate_threshold': '0.04%',
'complaint_rate_timestamp': '2018-06-17T02:11:25.787402+00:00',
'ts': '1529201485',
'version': 'v1.2018.06.18'
},
'group': 'aws-undefined',
'severity': 'critical',
'source': 'undefined-None-undefined-ses-account-monitor',
'summary': 'SES account reputation is at dangerous levels.',
'timestamp': '2018-06-17T02:11:25.787402+00:00'
},
'routing_key': None
}
assert len(result['slack']) == 1
assert result['slack'][0] == {
'attachments': [{
'color':
'danger',
'fallback':
'SES account reputation has breached CRITICAL threshold.',
'fields': [{
'short':
True,
'title':
'Service',
'value':
'<https://None.console.aws.amazon.com/ses/home?region=None#reputation-dashboard:|SES Account Reputation>'
}, {
'short': True,
'title': 'Account',
'value': 'undefined'
}, {
'short': True,
'title': 'Region',
'value': None
}, {
'short': True,
'title': 'Environment',
'value': 'undefined'
}, {
'short': True,
'title': 'Status',
'value': 'CRITICAL'
}, {
'short': True,
'title': 'Action',
'value': 'ALERT'
}, {
'short': True,
'title': 'Complaint Rate / Threshold',
'value': '99.00% / 0.04%'
}, {
'short': True,
'title': 'Complaint Rate Time',
'value': '2018-06-17T02:11:25.787402+00:00'
}, {
'short': True,
'title': 'Bounce Rate / Threshold',
'value': '5.00% / 5.00%'
}, {
'short': True,
'title': 'Bounce Rate Time',
'value': '2018-06-17T02:11:25.787402+00:00'
}, {
'short':
False,
'title':
'Message',
'value':
'SES account reputation has breached the CRITICAL threshold.'
}],
'footer':
'undefined-None-undefined-ses-account-monitor',
'footer_icon':
'https://platform.slack-edge.com/img/default_application_icon.png',
'ts':
1529201485
}],
'icon_emoji':
None,
'username':
'******'
}
def setUp(self):
self.cfn = boto3.client("cloudformation")
self.stubber = Stubber(self.cfn)
def test_create_medialive_channel(self):
"""Create an AWS medialive channel."""
key = "video-key"
medialive_input = {
"Input": {"Id": "input1"},
}
mediapackage_channel = {
"Id": "channel1",
"HlsIngest": {
"IngestEndpoints": [
{
"Id": "ingest1",
"Password": "******",
"Url": "https://endpoint1/channel",
"Username": "******",
},
{
"Id": "ingest2",
"Password": "******",
"Url": "https://endpoint2/channel",
"Username": "******",
},
]
},
}
medialive_channel_response = {"Channel": {"Id": "medialive_channel1"}}
with Stubber(medialive_utils.medialive_client) as medialive_stubber:
medialive_stubber.add_response(
"create_channel",
service_response=medialive_channel_response,
expected_params={
"InputSpecification": {
"Codec": "AVC",
"Resolution": "HD",
"MaximumBitrate": "MAX_10_MBPS",
},
"InputAttachments": [{"InputId": "input1"}],
"Destinations": [
{
"Id": "destination1",
"Settings": [
{
"PasswordParam": "test_user1",
"Url": "https://endpoint1/channel",
"Username": "******",
},
{
"PasswordParam": "test_user2",
"Url": "https://endpoint2/channel",
"Username": "******",
},
],
}
],
"Name": f"test_{key}",
"RoleArn": "medialive:role:arn",
"EncoderSettings": {
"AvailConfiguration": {
"AvailSettings": {
"Scte35SpliceInsert": {
"NoRegionalBlackoutFlag": "FOLLOW",
"WebDeliveryAllowedFlag": "FOLLOW",
}
}
},
"AudioDescriptions": [
{
"AudioSelectorName": "default",
"AudioTypeControl": "FOLLOW_INPUT",
"CodecSettings": {
"AacSettings": {
"Bitrate": 64000,
"RawFormat": "NONE",
"Spec": "MPEG4",
}
},
"LanguageCodeControl": "FOLLOW_INPUT",
"Name": "audio_1_aac64",
},
{
"AudioSelectorName": "default",
"AudioTypeControl": "FOLLOW_INPUT",
"CodecSettings": {
"AacSettings": {
"Bitrate": 96000,
"RawFormat": "NONE",
"Spec": "MPEG4",
}
},
"LanguageCodeControl": "FOLLOW_INPUT",
"Name": "audio_1_aac96",
},
{
"AudioSelectorName": "default",
"AudioTypeControl": "FOLLOW_INPUT",
"CodecSettings": {
"AacSettings": {
"Bitrate": 96000,
"RawFormat": "NONE",
"Spec": "MPEG4",
}
},
"LanguageCodeControl": "FOLLOW_INPUT",
"Name": "audio_2_aac96",
},
],
"OutputGroups": [
{
"Name": "TN2224",
"OutputGroupSettings": {
"HlsGroupSettings": {
"AdMarkers": ["ELEMENTAL_SCTE35"],
"CaptionLanguageMappings": [],
"CaptionLanguageSetting": "OMIT",
"ClientCache": "ENABLED",
"CodecSpecification": "RFC_4281",
"Destination": {
"DestinationRefId": "destination1"
},
"DirectoryStructure": "SINGLE_DIRECTORY",
"HlsCdnSettings": {
"HlsBasicPutSettings": {
"ConnectionRetryInterval": 30,
"FilecacheDuration": 300,
"NumRetries": 5,
"RestartDelay": 5,
}
},
"IndexNSegments": 15,
"InputLossAction": "EMIT_OUTPUT",
"IvInManifest": "INCLUDE",
"IvSource": "FOLLOWS_SEGMENT_NUMBER",
"KeepSegments": 21,
"ManifestCompression": "NONE",
"ManifestDurationFormat": "FLOATING_POINT",
"Mode": "LIVE",
"OutputSelection": "MANIFESTS_AND_SEGMENTS",
"ProgramDateTime": "INCLUDE",
"ProgramDateTimePeriod": 600,
"SegmentLength": 1,
"SegmentationMode": "USE_SEGMENT_DURATION",
"SegmentsPerSubdirectory": 10000,
"StreamInfResolution": "INCLUDE",
"TimedMetadataId3Frame": "PRIV",
"TimedMetadataId3Period": 10,
"TsFileMode": "SEGMENTED_FILES",
}
},
"Outputs": [
{
"AudioDescriptionNames": ["audio_1_aac96"],
"CaptionDescriptionNames": [],
"OutputSettings": {
"HlsOutputSettings": {
"HlsSettings": {
"StandardHlsSettings": {
"AudioRenditionSets": "PROGRAM_AUDIO",
"M3u8Settings": {
"AudioFramesPerPes": 4,
"AudioPids": "492-498",
"EcmPid": "8182",
"PcrControl": "PCR_EVERY_PES_PACKET",
"PmtPid": "480",
"ProgramNum": 1,
"Scte35Behavior": "PASSTHROUGH",
"Scte35Pid": "500",
"TimedMetadataPid": "502",
"TimedMetadataBehavior": "NO_PASSTH"
"ROUGH",
"VideoPid": "481",
},
}
},
"NameModifier": "_960x540_2000k",
}
},
"VideoDescriptionName": "video_854_480",
},
{
"AudioDescriptionNames": ["audio_2_aac96"],
"CaptionDescriptionNames": [],
"OutputSettings": {
"HlsOutputSettings": {
"HlsSettings": {
"StandardHlsSettings": {
"AudioRenditionSets": "PROGRAM_AUDIO",
"M3u8Settings": {
"AudioFramesPerPes": 4,
"AudioPids": "492-498",
"EcmPid": "8182",
"PcrControl": "PCR_EVERY_PES_PACKET",
"PmtPid": "480",
"ProgramNum": 1,
"Scte35Behavior": "PASSTHROUGH",
"Scte35Pid": "500",
"TimedMetadataPid": "502",
"TimedMetadataBehavior": "NO_PASSTH"
"ROUGH",
"VideoPid": "481",
},
}
},
"NameModifier": "_1280x720_3300k",
}
},
"VideoDescriptionName": "video_1280_720",
},
{
"AudioDescriptionNames": ["audio_1_aac64"],
"CaptionDescriptionNames": [],
"OutputSettings": {
"HlsOutputSettings": {
"HlsSettings": {
"StandardHlsSettings": {
"AudioRenditionSets": "PROGRAM_AUDIO",
"M3u8Settings": {
"AudioFramesPerPes": 4,
"AudioPids": "492-498",
"EcmPid": "8182",
"PcrControl": "PCR_EVERY_PES_PACKET",
"PmtPid": "480",
"ProgramNum": 1,
"Scte35Behavior": "PASSTHROUGH",
"Scte35Pid": "500",
"TimedMetadataPid": "502",
"TimedMetadataBehavior": "NO_PASSTH"
"ROUGH",
"VideoPid": "481",
},
}
},
"NameModifier": "_416x234_200k",
}
},
"VideoDescriptionName": "video_426_240",
},
],
}
],
"TimecodeConfig": {"Source": "SYSTEMCLOCK"},
"VideoDescriptions": [
{
"CodecSettings": {
"H264Settings": {
"AdaptiveQuantization": "HIGH",
"Bitrate": 350000,
"ColorMetadata": "INSERT",
"EntropyEncoding": "CAVLC",
"FlickerAq": "ENABLED",
"FramerateControl": "SPECIFIED",
"FramerateDenominator": 1100,
"FramerateNumerator": 30000,
"GopBReference": "DISABLED",
"GopNumBFrames": 2,
"GopSize": 1,
"GopSizeUnits": "SECONDS",
"Level": "H264_LEVEL_3",
"LookAheadRateControl": "HIGH",
"ParControl": "INITIALIZE_FROM_SOURCE",
"Profile": "MAIN",
"RateControlMode": "CBR",
"SceneChangeDetect": "ENABLED",
"SpatialAq": "ENABLED",
"Syntax": "DEFAULT",
"TemporalAq": "ENABLED",
"TimecodeInsertion": "DISABLED",
"NumRefFrames": 1,
"AfdSignaling": "NONE",
}
},
"Height": 240,
"Name": "video_426_240",
"ScalingBehavior": "DEFAULT",
"Width": 426,
"RespondToAfd": "NONE",
"Sharpness": 50,
},
{
"CodecSettings": {
"H264Settings": {
"AdaptiveQuantization": "HIGH",
"Bitrate": 1000000,
"ColorMetadata": "INSERT",
"EntropyEncoding": "CABAC",
"FlickerAq": "ENABLED",
"FramerateControl": "SPECIFIED",
"FramerateDenominator": 1100,
"FramerateNumerator": 30000,
"GopBReference": "ENABLED",
"GopNumBFrames": 2,
"GopSize": 1,
"GopSizeUnits": "SECONDS",
"Level": "H264_LEVEL_4_1",
"LookAheadRateControl": "HIGH",
"ParControl": "INITIALIZE_FROM_SOURCE",
"Profile": "HIGH",
"RateControlMode": "CBR",
"SceneChangeDetect": "ENABLED",
"SpatialAq": "ENABLED",
"Syntax": "DEFAULT",
"TemporalAq": "ENABLED",
"TimecodeInsertion": "DISABLED",
"NumRefFrames": 1,
"AfdSignaling": "NONE",
}
},
"Height": 480,
"Name": "video_854_480",
"ScalingBehavior": "DEFAULT",
"Width": 854,
"RespondToAfd": "NONE",
"Sharpness": 50,
},
{
"CodecSettings": {
"H264Settings": {
"AdaptiveQuantization": "HIGH",
"Bitrate": 3300000,
"ColorMetadata": "INSERT",
"EntropyEncoding": "CABAC",
"FlickerAq": "ENABLED",
"FramerateControl": "SPECIFIED",
"FramerateDenominator": 1100,
"FramerateNumerator": 30000,
"GopBReference": "ENABLED",
"GopNumBFrames": 2,
"GopSize": 1,
"GopSizeUnits": "SECONDS",
"Level": "H264_LEVEL_4_1",
"LookAheadRateControl": "HIGH",
"ParControl": "INITIALIZE_FROM_SOURCE",
"Profile": "HIGH",
"RateControlMode": "CBR",
"SceneChangeDetect": "ENABLED",
"SpatialAq": "ENABLED",
"Syntax": "DEFAULT",
"TemporalAq": "ENABLED",
"TimecodeInsertion": "DISABLED",
"NumRefFrames": 1,
"AfdSignaling": "NONE",
}
},
"Height": 720,
"Name": "video_1280_720",
"ScalingBehavior": "DEFAULT",
"Width": 1280,
"RespondToAfd": "NONE",
"Sharpness": 50,
},
],
},
"Tags": {"environment": "test", "app": "marsha"},
},
)
response = medialive_utils.create_medialive_channel(
key, medialive_input, mediapackage_channel
)
medialive_stubber.assert_no_pending_responses()
self.assertEqual(medialive_channel_response, response)
class TestS3Uploader(unittest.TestCase):
def setUp(self):
self.s3client = botocore.session.get_session().create_client(
's3', region_name="us-east-1")
self.s3client_stub = Stubber(self.s3client)
self.transfer_manager_mock = Mock(spec=S3Transfer)
self.transfer_manager_mock.upload = Mock()
self.bucket_name = "bucketname"
self.prefix = None
self.region = "us-east-1"
self.s3uploader = S3Uploader(
self.s3client, self.bucket_name, self.region, self.prefix, None, False,
self.transfer_manager_mock)
@patch('os.path.getsize', return_value=1)
@patch("awscli.customizations.s3uploader.ProgressPercentage")
def test_upload_successful(self, progress_percentage_mock, get_size_patch):
file_name = "filename"
remote_path = "remotepath"
prefix = "SomePrefix"
remote_path_with_prefix = "{0}/{1}".format(prefix, remote_path)
s3uploader = S3Uploader(
self.s3client, self.bucket_name, self.region, prefix, None, False,
self.transfer_manager_mock)
expected_upload_url = "s3://{0}/{1}/{2}".format(
self.bucket_name, prefix, remote_path)
# Setup mock to fake that file does not exist
s3uploader.file_exists = Mock()
s3uploader.file_exists.return_value = False
# set the metadata used by the uploader when uploading
artifact_metadata = {"key": "val"}
s3uploader.artifact_metadata = artifact_metadata
upload_url = s3uploader.upload(file_name, remote_path)
self.assertEquals(expected_upload_url, upload_url)
expected_extra_args = {
# expected encryption args
"ServerSideEncryption": "AES256",
# expected metadata
"Metadata": artifact_metadata
}
self.transfer_manager_mock.upload.assert_called_once_with(
file_name, self.bucket_name, remote_path_with_prefix,
expected_extra_args, mock.ANY)
s3uploader.file_exists.assert_called_once_with(remote_path_with_prefix)
@patch("awscli.customizations.s3uploader.ProgressPercentage")
def test_upload_idempotency(self, progress_percentage_mock):
file_name = "filename"
remote_path = "remotepath"
# Setup mock to fake that file was already uploaded
self.s3uploader.file_exists = Mock()
self.s3uploader.file_exists.return_value = True
self.s3uploader.upload(file_name, remote_path)
self.transfer_manager_mock.upload.assert_not_called()
self.s3uploader.file_exists.assert_called_once_with(remote_path)
@patch('os.path.getsize', return_value=1)
@patch("awscli.customizations.s3uploader.ProgressPercentage")
def test_upload_force_upload(self, progress_percentage_mock, get_size_patch):
file_name = "filename"
remote_path = "remotepath"
expected_upload_url = "s3://{0}/{1}".format(self.bucket_name,
remote_path)
# Set ForceUpload = True
self.s3uploader = S3Uploader(
self.s3client, self.bucket_name, self.region, self.prefix,
None, True, self.transfer_manager_mock)
# Pretend file already exists
self.s3uploader.file_exists = Mock()
self.s3uploader.file_exists.return_value = True
# Because we forced an update, this should reupload even if file exists
upload_url = self.s3uploader.upload(file_name, remote_path)
self.assertEquals(expected_upload_url, upload_url)
expected_encryption_args = {
"ServerSideEncryption": "AES256"
}
self.transfer_manager_mock.upload.assert_called_once_with(
file_name, self.bucket_name, remote_path,
expected_encryption_args, mock.ANY)
# Since ForceUpload=True, we should NEVER do the file-exists check
self.s3uploader.file_exists.assert_not_called()
@patch('os.path.getsize', return_value=1)
@patch("awscli.customizations.s3uploader.ProgressPercentage")
def test_upload_successful_custom_kms_key(self, progress_percentage_mock, get_size_patch):
file_name = "filename"
remote_path = "remotepath"
kms_key_id = "kms_id"
expected_upload_url = "s3://{0}/{1}".format(self.bucket_name,
remote_path)
# Set KMS Key Id
self.s3uploader = S3Uploader(
self.s3client, self.bucket_name, self.region, self.prefix,
kms_key_id, False, self.transfer_manager_mock)
# Setup mock to fake that file does not exist
self.s3uploader.file_exists = Mock()
self.s3uploader.file_exists.return_value = False
upload_url = self.s3uploader.upload(file_name, remote_path)
self.assertEquals(expected_upload_url, upload_url)
expected_encryption_args = {
"ServerSideEncryption": "aws:kms",
"SSEKMSKeyId": kms_key_id
}
self.transfer_manager_mock.upload.assert_called_once_with(
file_name, self.bucket_name, remote_path,
expected_encryption_args, mock.ANY)
self.s3uploader.file_exists.assert_called_once_with(remote_path)
@patch('os.path.getsize', return_value=1)
@patch("awscli.customizations.s3uploader.ProgressPercentage")
def test_upload_successful_nobucket(self, progress_percentage_mock, get_size_patch):
file_name = "filename"
remote_path = "remotepath"
# Setup mock to fake that file does not exist
self.s3uploader.file_exists = Mock()
self.s3uploader.file_exists.return_value = False
# Setup uploader to return a NOSuchBucket exception
exception = botocore.exceptions.ClientError(
{"Error": {"Code": "NoSuchBucket"}}, "OpName")
self.transfer_manager_mock.upload.side_effect = exception
with self.assertRaises(NoSuchBucketError):
self.s3uploader.upload(file_name, remote_path)
@patch('os.path.getsize', return_value=1)
@patch("awscli.customizations.s3uploader.ProgressPercentage")
def test_upload_successful_exceptions(self, progress_percentage_mock, get_size_patch):
file_name = "filename"
remote_path = "remotepath"
# Setup mock to fake that file does not exist
self.s3uploader.file_exists = Mock()
self.s3uploader.file_exists.return_value = False
# Raise an unrecognized botocore error
exception = botocore.exceptions.ClientError(
{"Error": {"Code": "SomeError"}}, "OpName")
self.transfer_manager_mock.upload.side_effect = exception
with self.assertRaises(botocore.exceptions.ClientError):
self.s3uploader.upload(file_name, remote_path)
# Some other exception
self.transfer_manager_mock.upload.side_effect = FloatingPointError()
with self.assertRaises(FloatingPointError):
self.s3uploader.upload(file_name, remote_path)
def test_upload_with_dedup(self):
checksum = "some md5 checksum"
filename = "filename"
extension = "extn"
self.s3uploader.file_checksum = Mock()
self.s3uploader.file_checksum.return_value = checksum
self.s3uploader.upload = Mock()
self.s3uploader.upload_with_dedup(filename, extension)
remotepath = "{0}.{1}".format(checksum, extension)
self.s3uploader.upload.assert_called_once_with(filename, remotepath)
def test_file_exists(self):
key = "some/path"
expected_params = {
"Bucket": self.bucket_name,
"Key": key
}
response = {
"AcceptRanges": "bytes",
"ContentType": "text/html",
"LastModified": "Thu, 16 Apr 2015 18:19:14 GMT",
"ContentLength": 77,
"VersionId": "null",
"ETag": "\"30a6ec7e1a9ad79c203d05a589c8b400\"",
"Metadata": {}
}
# Let's pretend file exists
self.s3client_stub.add_response("head_object",
response,
expected_params)
with self.s3client_stub:
self.assertTrue(self.s3uploader.file_exists(key))
# Let's pretend file does not exist
self.s3client_stub.add_client_error(
'head_object', "ClientError", "some error")
with self.s3client_stub:
self.assertFalse(self.s3uploader.file_exists(key))
# Let's pretend some other unknown exception happened
s3mock = Mock()
uploader = S3Uploader(s3mock, self.bucket_name, self.region)
s3mock.head_object = Mock()
s3mock.head_object.side_effect = RuntimeError()
with self.assertRaises(RuntimeError):
uploader.file_exists(key)
def test_file_checksum(self):
num_chars = 4096*5
data = ''.join(random.choice(string.ascii_uppercase)
for _ in range(num_chars)).encode('utf-8')
md5 = hashlib.md5()
md5.update(data)
expected_checksum = md5.hexdigest()
tempdir = tempfile.mkdtemp()
try:
filename = os.path.join(tempdir, 'tempfile')
with open(filename, 'wb') as f:
f.write(data)
actual_checksum = self.s3uploader.file_checksum(filename)
self.assertEqual(expected_checksum, actual_checksum)
finally:
shutil.rmtree(tempdir)
def test_make_url(self):
path = "Hello/how/are/you"
expected = "s3://{0}/{1}".format(self.bucket_name, path)
self.assertEquals(expected, self.s3uploader.make_url(path))
def test_to_path_style_s3_url_us_east_1(self):
key = "path/to/file"
version = "someversion"
region = "us-east-1"
s3uploader = S3Uploader(self.s3client, self.bucket_name, region)
result = s3uploader.to_path_style_s3_url(key, version)
self.assertEqual(
result,
"https://s3.amazonaws.com/{0}/{1}?versionId={2}".format(
self.bucket_name, key, version))
# Without versionId, that query parameter should be omitted
s3uploader = S3Uploader(self.s3client, self.bucket_name, region)
result = s3uploader.to_path_style_s3_url(key)
self.assertEqual(
result,
"https://s3.amazonaws.com/{0}/{1}".format(
self.bucket_name, key, version))
def test_to_path_style_s3_url_other_regions(self):
key = "path/to/file"
version = "someversion"
region = "us-west-2"
s3uploader = S3Uploader(self.s3client, self.bucket_name, region)
result = s3uploader.to_path_style_s3_url(key, version)
self.assertEqual(
result,
"https://s3-{0}.amazonaws.com/{1}/{2}?versionId={3}".format(
region, self.bucket_name, key, version))
# Without versionId, that query parameter should be omitted
s3uploader = S3Uploader(self.s3client, self.bucket_name, region)
result = s3uploader.to_path_style_s3_url(key)
self.assertEqual(
result,
"https://s3-{0}.amazonaws.com/{1}/{2}".format(
region, self.bucket_name, key))
def test_artifact_metadata_invalid_type(self):
prefix = "SomePrefix"
s3uploader = S3Uploader(
self.s3client, self.bucket_name, self.region, prefix, None, False,
self.transfer_manager_mock)
invalid_metadata = ["key", "val"]
with self.assertRaises(TypeError):
s3uploader.artifact_metadata = invalid_metadata
def setUp(self):
region = "us-east-1"
self.provider = Provider(region=region, recreate_failed=False)
self.stubber = Stubber(self.provider.cloudformation)
def forecast_stub():
client = boto3.client("sagemaker", region_name="us-east-1")
with Stubber(client) as stubber:
yield stubber
def setUp(self):
super(TestSTSPresignedUrl, self).setUp()
self.client = self.session.create_client('sts', 'us-west-2')
# Makes sure that no requests will go through
self.stubber = Stubber(self.client)
self.stubber.activate()
class TestOrganizationService(unittest.TestCase):
'''
Tests for cumulogenesis.services.organization.OrganizationService
'''
def setUp(self):
client = boto3.client('organizations')
self.stubber = Stubber(client)
self.stubber.activate()
base_session_mock = mock.Mock()
base_session_mock.client.return_value = client
self.session_builder_mock = mock.Mock()
self.session_builder_mock.get_base_session.return_value = base_session_mock
@staticmethod
def _get_mock_org(**kwargs):
return mock.Mock(spec=Organization, root_account_id='123456789', **kwargs)
def _get_org_service(self):
return OrganizationService(session_builder=self.session_builder_mock)
def test_load_organization_not_exist(self):
'''
Tests OrganizationService.load_organization when the organization doesn't exist
It should set organization.exists = False and should not call
_set_organization_attributes
'''
with mock.patch.object(OrganizationService, '_set_organization_attributes') as _set_attributes_mock:
org_service = self._get_org_service()
self.stubber.add_client_error('describe_organization')
org_mock = self._get_mock_org(exists=True)
org_service.load_organization(organization=org_mock)
assert org_mock.exists is False
_set_attributes_mock.assert_not_called()
def test_load_organization_exists(self):
'''
Tests OrganizationService.load_organization when the organization exists
It should set organization.exists = False and should not call
_set_organization_attributes
'''
with mock.patch.object(OrganizationService, '_set_organization_attributes') as _set_attributes_mock:
org_service = self._get_org_service()
describe_org_response = {
'Organization': {
"Id": "o-123456789",
"MasterAccountId": "123456789"}}
self.stubber.add_response('describe_organization', describe_org_response)
org_mock = self._get_mock_org(exists=True)
org_service.load_organization(organization=org_mock)
assert org_mock.exists is True
_set_attributes_mock.assert_called_with(org_model=org_mock,
describe_org_response=describe_org_response['Organization'])
def test_set_organization_attributes_not_master(self):
'''
Tests OrganizationService._set_organization_attributes when the account is not master
When the MasterAccountId for the organization does not match the root_account_id of
the organization model, cumulogenesis.exceptions.OrganizationMemberAccountException
should be raised
'''
with mock.patch.object(OrganizationService, '_get_root_parent_id'):
with mock.patch.object(OrganizationService, '_set_org_ids_to_children'):
org_service = self._get_org_service()
describe_org_response = {
'MasterAccountId': '987654321'}
org_mock = self._get_mock_org()
with self.assertRaises(exceptions.OrganizationMemberAccountException):
org_service._set_organization_attributes(org_model=org_mock,
describe_org_response=describe_org_response)
def test_set_organization_attributes(self):
'''
Tests OrganizationService._set_organization_attributes
'''
with mock.patch.object(OrganizationService, '_get_root_parent_id') as root_parent_id_mock:
with mock.patch.object(OrganizationService, '_set_org_ids_to_children'):
org_service = self._get_org_service()
describe_org_response = {
'MasterAccountId': '123456789',
'FeatureSet': 'ALL',
'Id': 'o-123456789'}
root_parent_id_mock.return_value = "r-1234"
org_mock = self._get_mock_org()
org_service._set_organization_attributes(org_model=org_mock,
describe_org_response=describe_org_response)
assert org_mock.featureset == 'ALL'
assert org_mock.org_id == 'o-123456789'
assert org_mock.root_parent_id == 'r-1234'
def test_get_root_parent_id(self):
'''
Tests OrganizationService._get_root_parent_id
'''
org_service = self._get_org_service()
list_parents_response = {
'Parents': [{
'Type': 'ROOT',
'Id': 'r-1234'}]}
self.stubber.add_response('list_parents', list_parents_response)
result = org_service._get_root_parent_id(root_account_id='123456789')
assert result == 'r-1234'
def test_set_org_ids_to_children(self):
'''
Tests OrganizationService._set_org_ids_to_children
This tests the enumeration for an effective organization hierarchy of:
r-1234:
orgunits:
ou-123456789:
accounts:
- 123456789
orgunits:
ou-987654321: {}
This test is sensitive to ordering and assumes that list_children will
first be called with ChildType=ORGANIZATIONAL_UNIT and then
ChildType=ACCOUNT in each iteration of the recursive function.
'''
child_orgunit_response1 = {"Children": [{"Id": "ou-123456789"}]}
child_orgunit_response2 = {"Children": [{"Id": "ou-987654321"}]}
child_account_response2 = {"Children": [{"Id": "123456789"}]}
child_empty_response = {"Children": []}
response_order = [
# First iteration: orgunit, account
child_orgunit_response1, child_empty_response,
# Second iteration
child_orgunit_response2, child_account_response2,
# Third iteration
child_empty_response, child_empty_response]
# Set the stub responses in order
for response in response_order:
self.stubber.add_response('list_children', response)
expected_ids_to_children = {
"r-1234": {"orgunits": ["ou-123456789"], "accounts": []},
"ou-123456789": {"orgunits": ["ou-987654321"], "accounts": ["123456789"]},
"ou-987654321": {"orgunits": [], "accounts": []}}
org_service = self._get_org_service()
org_mock = self._get_mock_org(ids_to_children={})
org_service._set_org_ids_to_children(org_model=org_mock, parent="r-1234")
helpers.print_expected_actual_diff(expected_ids_to_children, org_mock.ids_to_children)
assert expected_ids_to_children == org_mock.ids_to_children
def test_load_orgunits(self):
'''
Tests OrganizationService.load_orgunits
This tests the loading of orgunits for an effective organization hierarchy of:
r-1234:
orgunits:
ou-123456789:
accounts:
- 123456789
orgunits:
ou-987654321: {}
'''
ids_to_children_mock = {
"r-1234": {"orgunits": ["ou-123456789"], "accounts": []},
"ou-123456789": {"orgunits": ["ou-987654321"], "accounts": ["123456789"]},
"ou-987654321": {"orgunits": [], "accounts": []}}
desc_ou_response1 = {
"OrganizationalUnit": {
"Id": "ou-123456789", "Name": "orgunit_a"}}
desc_ou_response2 = {
"OrganizationalUnit": {
"Id": "ou-987654321", "Name": "orgunit_b"}}
for response in [desc_ou_response1, desc_ou_response2]:
self.stubber.add_response('describe_organizational_unit', response)
account_ids_to_names_mock = {"123456789": "account_a"}
orgunit_ids_to_names_mock = {
"ou-123456789": "orgunit_a",
"ou-987654321": "orgunit_b"}
expected_orgunits = {
"orgunit_a": {
"id": "ou-123456789", "name": "orgunit_a",
"child_orgunits": ["orgunit_b"], "accounts": ["account_a"]},
"orgunit_b": {
"id": "ou-987654321", "name": "orgunit_b",
"child_orgunits": [], "accounts": []}}
org_mock = self._get_mock_org(ids_to_children=ids_to_children_mock,
account_ids_to_names=account_ids_to_names_mock,
orgunit_ids_to_names=orgunit_ids_to_names_mock,
root_parent_id="r-1234", orgunits={})
org_service = self._get_org_service()
org_service.load_orgunits(organization=org_mock)
helpers.print_expected_actual_diff(expected_orgunits, org_mock.orgunits)
assert expected_orgunits == org_mock.orgunits
def test_load_policies(self):
'''
Tests OrganizationService.load_policies
'''
list_policies_response = {
"Policies": [
{"Id": "p-123456", "Name": "PolicyOne",
"Description": "The first policy", "AwsManaged": True}]}
describe_policy_response = {
"Policy": {
"Content": '{"This": "is a mock JSON document"}'}}
#pylint: disable=invalid-name
list_targets_for_policy_response = {
"Targets": [
{"Type": "Mock"}]}
self.stubber.add_response('list_policies', list_policies_response)
self.stubber.add_response('describe_policy', describe_policy_response)
self.stubber.add_response('list_targets_for_policy', list_targets_for_policy_response)
expected_document_content = OrderedDict({"This": "is a mock JSON document"})
expected_policies = {
"PolicyOne": {"id": "p-123456", "description": "The first policy",
"aws_managed": True, "name": "PolicyOne",
"document": {"content": expected_document_content}}}
org_mock = self._get_mock_org(policies={})
with mock.patch.object(OrganizationService, "_add_policy_to_target") as policy_to_target_mock:
org_service = self._get_org_service()
org_service.load_policies(organization=org_mock)
policy_to_target_mock.assert_called_with(
org_model=org_mock, target=list_targets_for_policy_response['Targets'][0],
policy_name="PolicyOne")
helpers.print_expected_actual_diff(expected_policies, org_mock.policies)
assert expected_policies == org_mock.policies
def test_add_policy_to_target_root(self):
'''
Tests OrganizationService._add_policy_to_target when the target type is ROOT
'''
mock_policy_name = "SomePolicy"
target_mock = {"Type": "ROOT"}
expected_policies = ["SomePolicy"]
org_mock = self._get_mock_org(root_policies=[])
org_service = self._get_org_service()
org_service._add_policy_to_target(org_model=org_mock, target=target_mock,
policy_name=mock_policy_name)
helpers.print_expected_actual_diff(expected_policies, org_mock.root_policies)
assert org_mock.root_policies == expected_policies
def test_add_policy_to_target_account(self):
'''
Tests OrganizationService._add_policy_to_target when the target type is ACCOUNT
'''
mock_policy_name = "SomePolicy"
target_mock = {"Type": "ACCOUNT", "Name": "account_a"}
mock_accounts = {"account_a": {}}
expected_accounts = {"account_a": {"policies": [mock_policy_name]}}
org_mock = self._get_mock_org(accounts=mock_accounts)
org_service = self._get_org_service()
org_service._add_policy_to_target(org_model=org_mock, target=target_mock,
policy_name=mock_policy_name)
helpers.print_expected_actual_diff(expected_accounts, org_mock.accounts)
assert expected_accounts == org_mock.accounts
def test_add_policy_to_target_orgunit(self):
'''
Tests OrganizationService._add_policy_to_target when the target type is ORGANIZATIONAL_UNIT
'''
mock_policy_name = "SomePolicy"
target_mock = {"Type": "ORGANIZATIONAL_UNIT", "Name": "orgunit_a"}
mock_orgunits = {"orgunit_a": {}}
expected_orgunits = {"orgunit_a": {"policies": [mock_policy_name]}}
org_mock = self._get_mock_org(orgunits=mock_orgunits)
org_service = self._get_org_service()
org_service._add_policy_to_target(org_model=org_mock, target=target_mock,
policy_name=mock_policy_name)
helpers.print_expected_actual_diff(expected_orgunits, org_mock.orgunits)
assert expected_orgunits == org_mock.orgunits
def test_load_accounts(self):
'''
Tests OrganizationService.load_accounts
'''
list_accounts_response = {
"Accounts": [{
"Name": "account_a", "Email": "*****@*****.**",
"Id": "123456789"}]}
self.stubber.add_response('list_accounts', list_accounts_response)
expected_accounts = {
"account_a": {
"name": "account_a", "owner": "*****@*****.**",
"id": "123456789", "regions": []}}
expected_account_ids_to_names = {
"123456789": "account_a"}
org_mock = self._get_mock_org(accounts={}, account_ids_to_names={})
org_service = self._get_org_service()
org_service.load_accounts(org_mock)
helpers.print_expected_actual_diff(expected_accounts, org_mock.accounts)
assert expected_accounts == org_mock.accounts
helpers.print_expected_actual_diff(expected_account_ids_to_names,
org_mock.account_ids_to_names)
assert expected_account_ids_to_names == org_mock.account_ids_to_names
def test_upsert_organization(self):
'''
Tests OrganizationService.upsert_organization when the action is create
'''
actions = {'organization': {'action': 'create'}}
list_parents_response = {
'Parents': [
{'Id': 'r-1234', 'Type': 'ROOT'}]}
expected_create_params = {'FeatureSet': 'ALL'}
expected_list_parents_params = {'ChildId': '123456789'}
expected_enable_policy_params = {
'RootId': 'r-1234', 'PolicyType': 'SERVICE_CONTROL_POLICY'}
self.stubber.add_response('create_organization', {'Organization': {}},
expected_create_params)
self.stubber.add_response('list_parents', list_parents_response,
expected_list_parents_params)
self.stubber.add_response('enable_policy_type', {'Root': {}},
expected_enable_policy_params)
expected_changes = {"organization": {"change": "created"}}
org_mock = self._get_mock_org(featureset='ALL', root_parent_id=None)
org_service = self._get_org_service()
changes = org_service.upsert_organization(organization=org_mock, actions=actions)
helpers.print_expected_actual_diff(expected_changes, changes)
assert expected_changes == changes
def test_upsert_organization_exists(self):
'''
Tests OrganizationService.upsert_organization when the action is not create
'''
actions = {'organization': {'action': 'update'}}
expected_changes = {}
org_mock = self._get_mock_org(featureset='ALL', root_parent_id=None)
org_service = self._get_org_service()
changes = org_service.upsert_organization(organization=org_mock, actions=actions)
helpers.print_expected_actual_diff(expected_changes, changes)
assert expected_changes == changes
def test_update_orgunit_policies(self):
'''
Tests OrganizationService.update_orgunit_policies with changes
'''
orgunit_mock = {'orgunit_a': {
"id": "ou-123456",
"policies": ["foo", "bar"]}}
aws_orgunit_mock = {'orgunit_a': {
"id": "ou-123456",
"policies": ["bar", "baz"]}}
aws_org_mock = self._get_mock_org(orgunits=aws_orgunit_mock)
org_mock = self._get_mock_org(orgunits=orgunit_mock, aws_model=aws_org_mock, updated_model=aws_org_mock)
with mock.patch.object(OrganizationService, 'update_entity_policy_attachments') as update_policy_mock:
org_service = self._get_org_service()
org_service.update_orgunit_policies(organization=org_mock, orgunit_name="orgunit_a")
print(update_policy_mock.call_args_list)
update_policy_mock.assert_called_with(
new_policies=["foo", "bar"], old_policies=["bar", "baz"],
org_model=org_mock, target_id="ou-123456")
def test_update_orgunit_policies_no_changes(self):
'''
Tests OrganizationService.update_orgunit_policies with no changes
'''
orgunit_mock = {'orgunit_a': {
"id": "ou-123456",
"policies": ["foo", "bar"]}}
aws_orgunit_mock = {'orgunit_a': {
"id": "ou-123456",
"policies": ["foo", "bar"]}}
aws_org_mock = self._get_mock_org(orgunits=aws_orgunit_mock)
org_mock = self._get_mock_org(orgunits=orgunit_mock, aws_model=aws_org_mock, updated_model=aws_org_mock)
with mock.patch.object(OrganizationService, 'update_entity_policy_attachments') as update_policy_mock:
org_service = self._get_org_service()
org_service.update_orgunit_policies(organization=org_mock, orgunit_name="orgunit_a")
update_policy_mock.assert_not_called()
def test_update_entity_policy_attachments(self):
'''
Tests OrganizationService.update_entity_policy_attachments
'''
target_id = 'ou-123456'
policies_mock = {
"policy_a": {'id': 'p-a'},
"policy_b": {'id': 'p-b'},
"policy_c": {'id': 'p-c'}}
old_policies = ["policy_a", "policy_b"]
new_policies = ["policy_b", "policy_c"]
expected_attach_parameters = {
"PolicyId": "p-c", "TargetId": "ou-123456"}
expected_detach_parameters = {
"PolicyId": "p-a", "TargetId": "ou-123456"}
self.stubber.add_response('attach_policy', {}, expected_attach_parameters)
self.stubber.add_response('detach_policy', {}, expected_detach_parameters)
updated_org_mock = self._get_mock_org(policies=policies_mock)
org_mock = self._get_mock_org(updated_model=updated_org_mock)
org_service = self._get_org_service()
org_service.update_entity_policy_attachments(
target_id=target_id, org_model=org_mock, old_policies=old_policies,
new_policies=new_policies)
def test_create_orgunit_root_parent(self):
'''
Tests OrganizationService.create_orgunit when the orgunit has the root as its parent
'''
create_orgunit_response = {
"OrganizationalUnit": {"Id": "ou-123456"}}
expected_create_orgunit_params = {"ParentId": "r-1234", "Name": "orgunit_a"}
self.stubber.add_response(
"create_organizational_unit", create_orgunit_response, expected_create_orgunit_params)
orgunits_mock = {"orgunit_a": {"name": "orgunit_a"}}
updated_org_mock = self._get_mock_org(root_parent_id='r-1234')
org_mock = self._get_mock_org(orgunits=orgunits_mock, updated_model=updated_org_mock)
org_service = self._get_org_service()
orgunit_id = org_service.create_orgunit(org_model=org_mock, orgunit_name="orgunit_a",
parent_name="root")
assert orgunit_id == "ou-123456"
def test_create_orgunit_orgunit_parent(self):
'''
Tests OrganizationService.create_orgunit when the orgunit has an orgunit as its parent
'''
create_orgunit_response = {
"OrganizationalUnit": {"Id": "ou-123456"}}
expected_create_orgunit_params = {"ParentId": "ou-654321", "Name": "orgunit_a"}
self.stubber.add_response(
"create_organizational_unit", create_orgunit_response, expected_create_orgunit_params)
orgunits_mock = {"orgunit_a": {"name": "orgunit_a"}}
updated_orgunits_mock = {"orgunit_b": {"name": "orgunit_b", "id": "ou-654321"}}
updated_org_mock = self._get_mock_org(root_parent_id='r-1234', orgunits=updated_orgunits_mock)
org_mock = self._get_mock_org(orgunits=orgunits_mock, updated_model=updated_org_mock)
org_service = self._get_org_service()
orgunit_id = org_service.create_orgunit(org_model=org_mock, orgunit_name="orgunit_a",
parent_name="orgunit_b")
assert orgunit_id == "ou-123456"
def test_upsert_policy_create(self):
'''
Tests OrganizationService.upsert_policy when the action is create
'''
action = {'action': 'create'}
policies_mock = {
"policy_a": {
"name": "policy_a",
"description": "policy_a description",
"document": {"content": {"foo": "bar"}}}}
expected_create_policy_params = {
"Content": '{"foo": "bar"}',
"Description": "policy_a description",
"Name": "policy_a",
"Type": "SERVICE_CONTROL_POLICY"}
create_policy_response = {
"Policy": {"PolicySummary": {"Id": "p-a"}}}
expected_changes = {"change": "created", "id": "p-a"}
self.stubber.add_response("create_policy", create_policy_response,
expected_create_policy_params)
org_mock = self._get_mock_org(policies=policies_mock)
org_service = self._get_org_service()
changes = org_service.upsert_policy(
organization=org_mock, policy_name="policy_a", action=action)
helpers.print_expected_actual_diff(expected_changes, changes)
assert changes == expected_changes
def test_upsert_policy_update(self):
'''
Tests OrganizationService.upsert_policy when the action is update
'''
action = {'action': 'update'}
updated_policies_mock = {
"policy_a": {
"id": "p-a",
"name": "policy_a",
"description": "policy_a description",
"document": {"content": {"foo": "bar"}}}}
policies_mock = {
"policy_a": {
"name": "policy_a",
"description": "policy_a description",
"document": {"content": {"foo": "bar"}}}}
expected_update_policy_params = {
"Content": '{"foo": "bar"}',
"Description": "policy_a description",
"Name": "policy_a",
"PolicyId": "p-a"}
update_policy_response = {
"Policy": {"PolicySummary": {"Id": "p-a"}}}
expected_changes = {"change": "updated", "id": "p-a"}
self.stubber.add_response("update_policy", update_policy_response,
expected_update_policy_params)
updated_org_mock = self._get_mock_org(policies=updated_policies_mock)
org_mock = self._get_mock_org(updated_model=updated_org_mock, policies=policies_mock)
org_service = self._get_org_service()
changes = org_service.upsert_policy(
organization=org_mock, policy_name="policy_a", action=action)
helpers.print_expected_actual_diff(expected_changes, changes)
assert changes == expected_changes
def test_delete_policy(self):
'''
Tests OrganizationService.delete_policy
'''
aws_policies_mock = {
"policy_a": {"id": "p-a"}}
expected_delete_policy_params = {"PolicyId": "p-a"}
expected_changes = {"change": "deleted", "id": "p-a"}
self.stubber.add_response("delete_policy", {}, expected_delete_policy_params)
aws_org_mock = self._get_mock_org(policies=aws_policies_mock)
org_mock = self._get_mock_org(aws_model=aws_org_mock)
org_service = self._get_org_service()
changes = org_service.delete_policy(organization=org_mock, policy_name="policy_a")
helpers.print_expected_actual_diff(expected_changes, changes)
assert changes == expected_changes
def test_delete_orgunit_error(self):
'''
Tests OrganizationService.delete_orgunit
When a client error is raised when deleting the Orgunit, the method
should return None, with the assumption that the Orgunit doesn't exist.
'''
self.stubber.add_client_error('delete_organizational_unit')
orgunit_mock = {"orgunit_a": {"id": "ou-123456"}}
aws_org_mock = self._get_mock_org(orgunits=orgunit_mock)
org_mock = self._get_mock_org(aws_model=aws_org_mock)
org_service = self._get_org_service()
result = org_service.delete_orgunit(organization=org_mock, orgunit_name="orgunit_a")
helpers.print_expected_actual_diff(None, result)
assert result is None
def test_delete_orgunit(self):
'''
Tests OrganizationService.delete_orgunit
'''
expected_delete_orgunit_params = {
"OrganizationalUnitId": "ou-123456"}
self.stubber.add_response('delete_organizational_unit', {},
expected_delete_orgunit_params)
orgunit_mock = {"orgunit_a": {"id": "ou-123456"}}
expected_changes = {"change": "deleted", "id": "ou-123456"}
aws_org_mock = self._get_mock_org(orgunits=orgunit_mock)
org_mock = self._get_mock_org(aws_model=aws_org_mock)
org_service = self._get_org_service()
changes = org_service.delete_orgunit(organization=org_mock, orgunit_name="orgunit_a")
helpers.print_expected_actual_diff(expected_changes, changes)
assert expected_changes == changes
#pylint: disable=too-many-locals
def test_create_accounts(self):
'''
Tests OrganizationService.create_accounts
Tests one each of a create account result status where the state is
SUCCEEDED, FAILED, or an unknown status.
'''
account_a_response = {
"CreateAccountStatus": {
"Id": "req-123456", "AccountName": "account_a", "State": "IN_PROGRESS"}}
account_b_response = {
"CreateAccountStatus": {
"Id": "req-654321", "AccountName": "account_b", "State": "IN_PROGRESS"}}
account_c_response = {
"CreateAccountStatus": {
"Id": "req-456789", "AccountName": "account_c", "State": "IN_PROGRESS"}}
account_a_expected_params = {
"AccountName": "account_a", "Email": "*****@*****.**"}
account_b_expected_params = {
"AccountName": "account_b", "Email": "*****@*****.**"}
account_c_expected_params = {
"AccountName": "account_c", "Email": "*****@*****.**"}
self.stubber.add_response('create_account', account_a_response, account_a_expected_params)
self.stubber.add_response('create_account', account_b_response, account_b_expected_params)
self.stubber.add_response('create_account', account_c_response, account_c_expected_params)
accounts_mock = {
"account_a": {"owner": "*****@*****.**", "name": "account_a"},
"account_b": {"owner": "*****@*****.**", "name": "account_b"},
"account_c": {"owner": "*****@*****.**", "name": "account_c"}}
accounts_to_create = ['account_a', 'account_b', 'account_c']
expected_changes = {
"account_a": {"change": "created"},
"account_b": {"change": "failed"},
"account_c": {"change": "unknown"}}
# self is just provided here as the method being mocked is an instance method
#pylint: disable=unused-argument
def _new_wait_on_account_creation(self, creation_statuses):
waiter_response = {
"account_a": {"State": "SUCCEEDED"},
"account_b": {"State": "FAILED"},
"account_c": {"State": "ERROR"}}
for account in creation_statuses:
creation_statuses[account] = waiter_response[account]
with mock.patch.object(OrganizationService, "_wait_on_account_creation",
new=_new_wait_on_account_creation):
org_mock = self._get_mock_org(accounts=accounts_mock)
org_service = self._get_org_service()
changes = org_service.create_accounts(organization=org_mock, accounts=accounts_to_create)
helpers.print_expected_actual_diff(expected_changes, changes)
assert changes == expected_changes
def test_wait_on_account_creation(self):
'''
Tests OrganizationService.wait_on_account_creation
'''
creation_statuses = {
"account_a": {"Id": "req-123456", "State": "IN_PROGRESS"}}
in_progress_response = {
"CreateAccountStatus": {"Id": "req-123456", "State": "IN_PROGRESS"}}
succeeded_response = {
"CreateAccountStatus": {"Id": "req-123456", "State": "SUCCEEDED"}}
expected_describe_status_params = {"CreateAccountRequestId": "req-123456"}
self.stubber.add_response("describe_create_account_status", in_progress_response,
expected_describe_status_params)
self.stubber.add_response("describe_create_account_status", succeeded_response,
expected_describe_status_params)
expected_creation_statuses = {
"account_a": {"Id": "req-123456", "State": "SUCCEEDED"}}
# Patch time.sleep so we don't actually wait 20 seconds for this test to complete.
with mock.patch('cumulogenesis.services.organization.time.sleep'):
org_service = self._get_org_service()
org_service._wait_on_account_creation(creation_statuses)
helpers.print_expected_actual_diff(expected_creation_statuses, creation_statuses)
assert expected_creation_statuses == creation_statuses
def test_move_account_root(self):
'''
Tests OrganizationService.move_account when the parent_name is root
'''
updated_accounts_mock = {"account_a": {"id": "987654321"}}
expected_changes = {"changes": "reassociated", "parent": "r-1234"}
list_parents_response = {
"Parents": [{"Id": "ou-123456"}]}
expected_list_parents_params = {"ChildId": "987654321"}
expected_move_account_params = {
"AccountId": "987654321", "SourceParentId": "ou-123456",
"DestinationParentId": "r-1234"}
self.stubber.add_response("list_parents", list_parents_response,
expected_list_parents_params)
self.stubber.add_response("move_account", {}, expected_move_account_params)
updated_org_mock = self._get_mock_org(accounts=updated_accounts_mock)
aws_org_mock = self._get_mock_org(root_parent_id='r-1234')
org_mock = self._get_mock_org(aws_model=aws_org_mock, updated_model=updated_org_mock)
org_service = self._get_org_service()
changes = org_service.move_account(organization=org_mock, account_name="account_a",
parent_name="root")
helpers.print_expected_actual_diff(expected_changes, changes)
assert changes == expected_changes
def test_move_account_no_change(self):
'''
Tests OrganizationService.move_account when the source and destination IDs match
'''
updated_accounts_mock = {"account_a": {"id": "987654321"}}
expected_changes = {}
list_parents_response = {
"Parents": [{"Id": "r-1234"}]}
expected_list_parents_params = {"ChildId": "987654321"}
self.stubber.add_response("list_parents", list_parents_response,
expected_list_parents_params)
updated_org_mock = self._get_mock_org(accounts=updated_accounts_mock)
aws_org_mock = self._get_mock_org(root_parent_id='r-1234')
org_mock = self._get_mock_org(aws_model=aws_org_mock, updated_model=updated_org_mock)
org_service = self._get_org_service()
changes = org_service.move_account(organization=org_mock, account_name="account_a",
parent_name="root")
helpers.print_expected_actual_diff(expected_changes, changes)
assert changes == expected_changes
def test_move_account_orgunit(self):
'''
Tests OrganizationService.move_account when the parent_name is an orgunit
(not "root")
'''
updated_accounts_mock = {"account_a": {"id": "987654321"}}
updated_orgunits_mock = {"orgunit_a": {"id": "ou-654321"}}
expected_changes = {"changes": "reassociated", "parent": "ou-654321"}
list_parents_response = {
"Parents": [{"Id": "ou-123456"}]}
expected_list_parents_params = {"ChildId": "987654321"}
expected_move_account_params = {
"AccountId": "987654321", "SourceParentId": "ou-123456",
"DestinationParentId": "ou-654321"}
self.stubber.add_response("list_parents", list_parents_response,
expected_list_parents_params)
self.stubber.add_response("move_account", {}, expected_move_account_params)
updated_org_mock = self._get_mock_org(accounts=updated_accounts_mock,
orgunits=updated_orgunits_mock)
aws_org_mock = self._get_mock_org()
org_mock = self._get_mock_org(aws_model=aws_org_mock, updated_model=updated_org_mock)
org_service = self._get_org_service()
changes = org_service.move_account(organization=org_mock, account_name="account_a",
parent_name="orgunit_a")
helpers.print_expected_actual_diff(expected_changes, changes)
assert changes == expected_changes
def _stubbed_cf_client(self):
cf = botocore.session.get_session().create_client(
"cloudformation", region_name="us-west-2")
return [cf, Stubber(cf)]
#!/usr/bin/env python
import unittest
import json
from botocore.stub import Stubber
from functions import file as file_functions
from models import file as file_models
from lib import utils
import runtime_context
# https://botocore.amazonaws.com/v1/documentation/api/latest/reference/stubber.html
ddb_stubber = Stubber(file_models.DDB_CLIENT)
s3_stubber = Stubber(file_functions.S3_CLIENT)
def generate_id():
return 'abcd'
utils.generate_id = generate_id
class FileTest(unittest.TestCase):
def test_preprocess(self):
# setting the environment variables
runtime_context.BUCKET_NAME = 'test-bucket'
runtime_context.EXPIRATION = 3600
runtime_context.STORE = False
ddb_expected = {
'TableName': file_models.FileModel._TABLE_NAME,
'Item': {
def setUp(self):
region = "us-east-1"
self.provider = Provider(region=region,
interactive=True,
recreate_failed=True)
self.stubber = Stubber(self.provider.cloudformation)
class TestSSMStoreHandler(unittest.TestCase):
"""Tests for runway.cfngin.lookups.handlers.ssmstore.SsmstoreLookup."""
client = boto3.client(
'ssm',
region_name='us-east-1',
# bypass the need to have these in the env
aws_access_key_id='testing',
aws_secret_access_key='testing')
def setUp(self):
"""Run before tests."""
self.stubber = Stubber(self.client)
self.get_parameters_response = {
'Parameters': [{
'Name': 'ssmkey',
'Type': 'String',
'Value': 'ssmvalue'
}],
'InvalidParameters': ['invalid_ssm_param']
}
self.invalid_get_parameters_response = {
'InvalidParameters': ['ssmkey']
}
self.expected_params = {'Names': ['ssmkey'], 'WithDecryption': True}
self.ssmkey = "ssmkey"
self.ssmvalue = "ssmvalue"
@mock.patch('runway.cfngin.lookups.handlers.ssmstore.get_session',
return_value=SessionStub(client))
def test_ssmstore_handler(self, _mock_client):
"""Test ssmstore handler."""
self.stubber.add_response('get_parameters',
self.get_parameters_response,
self.expected_params)
with self.stubber:
value = SsmstoreLookup.handle(self.ssmkey)
self.assertEqual(value, self.ssmvalue)
self.assertIsInstance(value, string_types)
@mock.patch('runway.cfngin.lookups.handlers.ssmstore.get_session',
return_value=SessionStub(client))
def test_ssmstore_invalid_value_handler(self, _mock_client):
"""Test ssmstore invalid value handler."""
self.stubber.add_response('get_parameters',
self.invalid_get_parameters_response,
self.expected_params)
with self.stubber:
try:
SsmstoreLookup.handle(self.ssmkey)
except ValueError:
assert True
@mock.patch('runway.cfngin.lookups.handlers.ssmstore.get_session',
return_value=SessionStub(client))
def test_ssmstore_handler_with_region(self, _mock_client):
"""Test ssmstore handler with region."""
self.stubber.add_response('get_parameters',
self.get_parameters_response,
self.expected_params)
region = "us-east-1"
temp_value = "%s@%s" % (region, self.ssmkey)
with self.stubber:
value = SsmstoreLookup.handle(temp_value)
self.assertEqual(value, self.ssmvalue)
class TestProviderInteractiveMode(unittest.TestCase):
def setUp(self):
region = "us-east-1"
self.provider = Provider(region=region,
interactive=True,
recreate_failed=True)
self.stubber = Stubber(self.provider.cloudformation)
def test_successful_init(self):
region = "us-east-1"
replacements = True
p = Provider(region=region,
interactive=True,
replacements_only=replacements)
self.assertEqual(p.region, region)
self.assertEqual(p.replacements_only, replacements)
@patch("stacker.providers.aws.default.ask_for_approval")
def test_update_stack_execute_success(self, patched_approval):
stack_name = "my-fake-stack"
self.stubber.add_response("create_change_set", {
'Id': 'CHANGESETID',
'StackId': 'STACKID'
})
changes = []
changes.append(generate_change())
self.stubber.add_response(
"describe_change_set",
generate_change_set_response(
status="CREATE_COMPLETE",
execution_status="AVAILABLE",
changes=changes,
))
self.stubber.add_response("execute_change_set", {})
with self.stubber:
self.provider.update_stack(
fqn=stack_name,
template=Template(url="http://fake.template.url.com/"),
old_parameters=[],
parameters=[],
tags=[])
patched_approval.assert_called_with(full_changeset=changes,
params_diff=[],
include_verbose=True)
self.assertEqual(patched_approval.call_count, 1)
def test_select_update_method(self):
for i in [[{
'force_interactive': False,
'force_change_set': False
}, self.provider.interactive_update_stack],
[{
'force_interactive': True,
'force_change_set': False
}, self.provider.interactive_update_stack],
[{
'force_interactive': False,
'force_change_set': True
}, self.provider.interactive_update_stack],
[{
'force_interactive': True,
'force_change_set': True
}, self.provider.interactive_update_stack]]:
self.assertEquals(self.provider.select_update_method(**i[0]), i[1])
def s3_stubber():
"""S3 stubber using the botocore Stubber class."""
s3_client = get_s3_client_for_bucket('default')
with Stubber(s3_client) as s3_stubber:
yield s3_stubber
class TestProviderDefaultMode(unittest.TestCase):
def setUp(self):
region = "us-east-1"
self.provider = Provider(region=region, recreate_failed=False)
self.stubber = Stubber(self.provider.cloudformation)
def test_get_stack_stack_does_not_exist(self):
stack_name = "MockStack"
self.stubber.add_client_error(
"describe_stacks",
service_error_code="ValidationError",
service_message="Stack with id %s does not exist" % stack_name,
expected_params={"StackName": stack_name})
with self.assertRaises(exceptions.StackDoesNotExist):
with self.stubber:
self.provider.get_stack(stack_name)
def test_get_stack_stack_exists(self):
stack_name = "MockStack"
stack_response = {
"Stacks": [generate_describe_stacks_stack(stack_name)]
}
self.stubber.add_response("describe_stacks",
stack_response,
expected_params={"StackName": stack_name})
with self.stubber:
response = self.provider.get_stack(stack_name)
self.assertEqual(response["StackName"], stack_name)
def test_select_update_method(self):
for i in [[{
'force_interactive': True,
'force_change_set': False
}, self.provider.interactive_update_stack],
[{
'force_interactive': False,
'force_change_set': False
}, self.provider.default_update_stack],
[{
'force_interactive': False,
'force_change_set': True
}, self.provider.noninteractive_changeset_update],
[{
'force_interactive': True,
'force_change_set': True
}, self.provider.interactive_update_stack]]:
self.assertEquals(self.provider.select_update_method(**i[0]), i[1])
def test_prepare_stack_for_update_missing(self):
stack_name = "MockStack"
self.stubber.add_client_error(
"describe_stacks",
service_error_code="ValidationError",
service_message="Stack with id %s does not exist" % stack_name,
expected_params={"StackName": stack_name})
with self.assertRaises(exceptions.StackDoesNotExist):
with self.stubber:
self.provider.prepare_stack_for_update(stack_name, [])
def test_prepare_stack_for_update_completed(self):
stack_name = "MockStack"
stack_response = {
"Stacks": [
generate_describe_stacks_stack(stack_name,
stack_status="UPDATE_COMPLETE")
]
}
self.stubber.add_response("describe_stacks",
stack_response,
expected_params={"StackName": stack_name})
with self.stubber:
self.assertTrue(
self.provider.prepare_stack_for_update(stack_name, []))
def test_prepare_stack_for_update_in_progress(self):
stack_name = "MockStack"
stack_response = {
"Stacks": [
generate_describe_stacks_stack(
stack_name, stack_status="UPDATE_IN_PROGRESS")
]
}
self.stubber.add_response("describe_stacks",
stack_response,
expected_params={"StackName": stack_name})
with self.assertRaises(exceptions.StackUpdateBadStatus) as raised:
with self.stubber:
self.provider.prepare_stack_for_update(stack_name, [])
self.assertIn('in-progress', raised.exception.message)
def test_prepare_stack_for_update_non_recreatable(self):
stack_name = "MockStack"
stack_response = {
"Stacks": [
generate_describe_stacks_stack(
stack_name, stack_status="REVIEW_IN_PROGRESS")
]
}
self.stubber.add_response("describe_stacks",
stack_response,
expected_params={"StackName": stack_name})
with self.assertRaises(exceptions.StackUpdateBadStatus) as raised:
with self.stubber:
self.provider.prepare_stack_for_update(stack_name, [])
self.assertIn('Unsupported state', raised.exception.message)
def test_prepare_stack_for_update_disallowed(self):
stack_name = "MockStack"
stack_response = {
"Stacks": [
generate_describe_stacks_stack(
stack_name, stack_status="ROLLBACK_COMPLETE")
]
}
self.stubber.add_response("describe_stacks",
stack_response,
expected_params={"StackName": stack_name})
with self.assertRaises(exceptions.StackUpdateBadStatus) as raised:
with self.stubber:
self.provider.prepare_stack_for_update(stack_name, [])
self.assertIn('re-creation is disabled', raised.exception.message)
# Ensure we point out to the user how to enable re-creation
self.assertIn('--recreate-failed', raised.exception.message)
def test_prepare_stack_for_update_bad_tags(self):
stack_name = "MockStack"
stack_response = {
"Stacks": [
generate_describe_stacks_stack(
stack_name, stack_status="ROLLBACK_COMPLETE")
]
}
self.stubber.add_response("describe_stacks",
stack_response,
expected_params={"StackName": stack_name})
self.provider.recreate_failed = True
with self.assertRaises(exceptions.StackUpdateBadStatus) as raised:
with self.stubber:
self.provider.prepare_stack_for_update(stack_name,
tags=[{
'Key':
'stacker_namespace',
'Value': 'test'
}])
self.assertIn('tags differ', raised.exception.message.lower())
def test_prepare_stack_for_update_recreate(self):
stack_name = "MockStack"
stack_response = {
"Stacks": [
generate_describe_stacks_stack(
stack_name, stack_status="ROLLBACK_COMPLETE")
]
}
self.stubber.add_response("describe_stacks",
stack_response,
expected_params={"StackName": stack_name})
self.stubber.add_response("delete_stack", {},
expected_params={"StackName": stack_name})
self.provider.recreate_failed = True
with self.stubber:
self.assertFalse(
self.provider.prepare_stack_for_update(stack_name, []))
def test_send_notifications_critical(monitor, end_datetime,
metric_data_results_response_critical,
metric_data_results_params):
ses_stubber = Stubber(monitor.ses_service.client)
ses_stubber.add_response('get_send_quota', {
'Max24HourSend': 10.0,
'MaxSendRate': 523.0,
'SentLast24Hours': 15.0
}, {})
ses_stubber.activate()
cloudwatch_stubber = Stubber(monitor.cloudwatch_service.client)
cloudwatch_stubber.add_response('get_metric_data',
metric_data_results_response_critical,
metric_data_results_params)
cloudwatch_stubber.activate()
with responses.RequestsMock(
target='botocore.vendored.requests.adapters.HTTPAdapter.send'
) as rsps:
rsps.add(responses.POST,
monitor.pager_duty_service.url,
status=202,
json={
'status': 'success',
'message': 'Event processed',
'dedup_key': 'samplekeyhere'
})
rsps.add(responses.POST,
monitor.slack_service.url,
status=200,
json={'ok': True})
monitor.handle_ses_sending_quota(target_datetime=end_datetime)
monitor.handle_ses_reputation(target_datetime=end_datetime)
result = monitor.send_notifications(raise_on_errors=True)
assert len(result['pager_duty']) == 2
assert len(result['slack']) == 2
class TestMethods(unittest.TestCase):
def setUp(self):
self.cfn = boto3.client("cloudformation")
self.stubber = Stubber(self.cfn)
def test_requires_replacement(self):
changeset = [
generate_resource_change(),
generate_resource_change(replacement=False),
generate_resource_change(),
]
replacement = requires_replacement(changeset)
self.assertEqual(len(replacement), 2)
for resource in replacement:
self.assertEqual(resource["ResourceChange"]["Replacement"], "True")
def test_summarize_params_diff(self):
unmodified_param = DictValue("ParamA", "new-param-value",
"new-param-value")
modified_param = DictValue("ParamB", "param-b-old-value",
"param-b-new-value-delta")
added_param = DictValue("ParamC", None, "param-c-new-value")
removed_param = DictValue("ParamD", "param-d-old-value", None)
params_diff = [
unmodified_param,
modified_param,
added_param,
removed_param,
]
self.assertEqual(summarize_params_diff([]), "")
self.assertEqual(
summarize_params_diff(params_diff), '\n'.join([
"Parameters Added: ParamC",
"Parameters Removed: ParamD",
"Parameters Modified: ParamB\n",
]))
only_modified_params_diff = [modified_param]
self.assertEqual(summarize_params_diff(only_modified_params_diff),
"Parameters Modified: ParamB\n")
only_added_params_diff = [added_param]
self.assertEqual(summarize_params_diff(only_added_params_diff),
"Parameters Added: ParamC\n")
only_removed_params_diff = [removed_param]
self.assertEqual(summarize_params_diff(only_removed_params_diff),
"Parameters Removed: ParamD\n")
@patch("stacker.providers.aws.default.format_params_diff")
def test_ask_for_approval(self, patched_format):
get_input_path = "stacker.providers.aws.default.get_raw_input"
with patch(get_input_path, return_value="y"):
self.assertIsNone(ask_for_approval([], [], None))
for v in ("n", "N", "x", "\n"):
with patch(get_input_path, return_value=v):
with self.assertRaises(exceptions.CancelExecution):
ask_for_approval([], [])
with patch(get_input_path, side_effect=["v", "n"]) as mock_get_input:
with patch("yaml.safe_dump") as mock_safe_dump:
with self.assertRaises(exceptions.CancelExecution):
ask_for_approval([], [], True)
self.assertEqual(mock_safe_dump.call_count, 1)
self.assertEqual(mock_get_input.call_count, 2)
self.assertEqual(patched_format.call_count, 0)
@patch("stacker.providers.aws.default.format_params_diff")
def test_ask_for_approval_with_params_diff(self, patched_format):
get_input_path = "stacker.providers.aws.default.get_raw_input"
params_diff = [
DictValue('ParamA', None, 'new-param-value'),
DictValue('ParamB', 'param-b-old-value', 'param-b-new-value-delta')
]
with patch(get_input_path, return_value="y"):
self.assertIsNone(ask_for_approval([], params_diff, None))
for v in ("n", "N", "x", "\n"):
with patch(get_input_path, return_value=v):
with self.assertRaises(exceptions.CancelExecution):
ask_for_approval([], params_diff)
with patch(get_input_path, side_effect=["v", "n"]) as mock_get_input:
with patch("yaml.safe_dump") as mock_safe_dump:
with self.assertRaises(exceptions.CancelExecution):
ask_for_approval([], params_diff, True)
self.assertEqual(mock_safe_dump.call_count, 1)
self.assertEqual(mock_get_input.call_count, 2)
self.assertEqual(patched_format.call_count, 1)
def test_wait_till_change_set_complete_success(self):
self.stubber.add_response(
"describe_change_set",
generate_change_set_response("CREATE_COMPLETE"))
with self.stubber:
wait_till_change_set_complete(self.cfn, "FAKEID")
self.stubber.add_response("describe_change_set",
generate_change_set_response("FAILED"))
with self.stubber:
wait_till_change_set_complete(self.cfn, "FAKEID")
def test_wait_till_change_set_complete_failed(self):
# Need 2 responses for try_count
for i in range(2):
self.stubber.add_response(
"describe_change_set",
generate_change_set_response("CREATE_PENDING"))
with self.stubber:
with self.assertRaises(exceptions.ChangesetDidNotStabilize):
wait_till_change_set_complete(self.cfn,
"FAKEID",
try_count=2,
sleep_time=.1)
def test_create_change_set_stack_did_not_change(self):
self.stubber.add_response("create_change_set", {
'Id': 'CHANGESETID',
'StackId': 'STACKID'
})
self.stubber.add_response(
"describe_change_set",
generate_change_set_response(
"FAILED", status_reason="Stack didn't contain changes."))
self.stubber.add_response(
"delete_change_set", {},
expected_params={"ChangeSetName": "CHANGESETID"})
with self.stubber:
with self.assertRaises(exceptions.StackDidNotChange):
create_change_set(
cfn_client=self.cfn,
fqn="my-fake-stack",
template=Template(url="http://fake.template.url.com/"),
parameters=[],
tags=[])
def test_create_change_set_unhandled_failed_status(self):
self.stubber.add_response("create_change_set", {
'Id': 'CHANGESETID',
'StackId': 'STACKID'
})
self.stubber.add_response(
"describe_change_set",
generate_change_set_response(
"FAILED", status_reason="Some random bad thing."))
with self.stubber:
with self.assertRaises(exceptions.UnhandledChangeSetStatus):
create_change_set(
cfn_client=self.cfn,
fqn="my-fake-stack",
template=Template(url="http://fake.template.url.com/"),
parameters=[],
tags=[])
def test_create_change_set_bad_execution_status(self):
self.stubber.add_response("create_change_set", {
'Id': 'CHANGESETID',
'StackId': 'STACKID'
})
self.stubber.add_response(
"describe_change_set",
generate_change_set_response(
status="CREATE_COMPLETE",
execution_status="UNAVAILABLE",
))
with self.stubber:
with self.assertRaises(exceptions.UnableToExecuteChangeSet):
create_change_set(
cfn_client=self.cfn,
fqn="my-fake-stack",
template=Template(url="http://fake.template.url.com/"),
parameters=[],
tags=[])
def test_generate_cloudformation_args(self):
stack_name = "mystack"
template_url = "http://fake.s3url.com/blah.json"
template_body = '{"fake_body": "woot"}'
std_args = {
"stack_name": stack_name,
"parameters": [],
"tags": [],
"template": Template(url=template_url)
}
std_return = {
"StackName": stack_name,
"Parameters": [],
"Tags": [],
"Capabilities": DEFAULT_CAPABILITIES,
"TemplateURL": template_url
}
result = generate_cloudformation_args(**std_args)
self.assertEqual(result, std_return)
result = generate_cloudformation_args(service_role="FakeRole",
**std_args)
service_role_result = copy.deepcopy(std_return)
service_role_result["RoleARN"] = "FakeRole"
self.assertEqual(result, service_role_result)
result = generate_cloudformation_args(change_set_name="MyChanges",
**std_args)
change_set_result = copy.deepcopy(std_return)
change_set_result["ChangeSetName"] = "MyChanges"
self.assertEqual(result, change_set_result)
# If not TemplateURL is provided, use TemplateBody
std_args["template"] = Template(body=template_body)
template_body_result = copy.deepcopy(std_return)
del (template_body_result["TemplateURL"])
template_body_result["TemplateBody"] = template_body
result = generate_cloudformation_args(**std_args)
self.assertEqual(result, template_body_result)
def test_handle_ses_reputation_warning(monitor, end_datetime,
metric_data_results_response_warning,
metric_data_results_params):
cloudwatch_stubber = Stubber(monitor.cloudwatch_service.client)
cloudwatch_stubber.add_response('get_metric_data',
metric_data_results_response_warning,
metric_data_results_params)
cloudwatch_stubber.activate()
result = monitor.handle_ses_reputation(target_datetime=end_datetime)
assert result['pager_duty'] == deque([])
assert len(result['slack']) == 1
assert result['slack'][0] == {
'attachments': [{
'color':
'warning',
'fallback':
'SES account reputation has breached WARNING threshold.',
'fields': [{
'short':
True,
'title':
'Service',
'value':
'<https://None.console.aws.amazon.com/ses/home?region=None#reputation-dashboard:|SES Account Reputation>'
}, {
'short': True,
'title': 'Account',
'value': 'undefined'
}, {
'short': True,
'title': 'Region',
'value': None
}, {
'short': True,
'title': 'Environment',
'value': 'undefined'
}, {
'short': True,
'title': 'Status',
'value': 'WARNING'
}, {
'short': True,
'title': 'Action',
'value': 'ALERT'
}, {
'short': True,
'title': 'Bounce Rate / Threshold',
'value': '5.24% / 5.00%'
}, {
'short': True,
'title': 'Bounce Rate Time',
'value': '2018-06-17T02:11:25.787402+00:00'
}, {
'short':
False,
'title':
'Message',
'value':
'SES account reputation has breached the WARNING threshold.'
}],
'footer':
'undefined-None-undefined-ses-account-monitor',
'footer_icon':
'https://platform.slack-edge.com/img/default_application_icon.png',
'ts':
1529201485
}],
'icon_emoji':
None,
'username':
'******'
}
class TestDisplayInfected(unittest.TestCase):
def setUp(self):
self.s3_bucket_name = "test_bucket"
self.s3_client = botocore.session.get_session().create_client("s3")
self.stubber = Stubber(self.s3_client)
list_objects_v2_response = {
"IsTruncated":
False,
"Contents": [{
"Key": "test.txt",
"LastModified": datetime.datetime(2015, 1, 1),
"ETag": '"abc123"',
"Size": 123,
"StorageClass": "STANDARD",
"Owner": {
"DisplayName": "myname",
"ID": "abc123"
},
}],
"Name":
self.s3_bucket_name,
"Prefix":
"",
"MaxKeys":
1000,
"EncodingType":
"url",
}
list_objects_v2_expected_params = {"Bucket": self.s3_bucket_name}
self.stubber.add_response("list_objects_v2", list_objects_v2_response,
list_objects_v2_expected_params)
def test_get_objects_previously_scanned_status(self):
get_object_tagging_response = {
"VersionId": "abc123",
"TagSet": [{
"Key": AV_STATUS_METADATA,
"Value": AV_STATUS_INFECTED
}],
}
get_object_tagging_expected_params = {
"Bucket": self.s3_bucket_name,
"Key": "test.txt",
}
self.stubber.add_response(
"get_object_tagging",
get_object_tagging_response,
get_object_tagging_expected_params,
)
with self.stubber:
s3_object_list = get_objects(self.s3_client, self.s3_bucket_name)
expected_object_list = []
self.assertEqual(s3_object_list, expected_object_list)
def test_get_objects_previously_scanned_timestamp(self):
get_object_tagging_response = {
"VersionId": "abc123",
"TagSet": [{
"Key": AV_TIMESTAMP_METADATA,
"Value": get_timestamp()
}],
}
get_object_tagging_expected_params = {
"Bucket": self.s3_bucket_name,
"Key": "test.txt",
}
self.stubber.add_response(
"get_object_tagging",
get_object_tagging_response,
get_object_tagging_expected_params,
)
with self.stubber:
s3_object_list = get_objects(self.s3_client, self.s3_bucket_name)
expected_object_list = []
self.assertEqual(s3_object_list, expected_object_list)
def test_get_objects_unscanned(self):
get_object_tagging_response = {"VersionId": "abc123", "TagSet": []}
get_object_tagging_expected_params = {
"Bucket": self.s3_bucket_name,
"Key": "test.txt",
}
self.stubber.add_response(
"get_object_tagging",
get_object_tagging_response,
get_object_tagging_expected_params,
)
with self.stubber:
s3_object_list = get_objects(self.s3_client, self.s3_bucket_name)
expected_object_list = ["test.txt"]
self.assertEqual(s3_object_list, expected_object_list)
def test_format_s3_event(self):
key_name = "key"
s3_event = format_s3_event(self.s3_bucket_name, key_name)
expected_s3_event = {
"Records": [{
"s3": {
"bucket": {
"name": self.s3_bucket_name
},
"object": {
"key": key_name
},
}
}]
}
self.assertEquals(s3_event, expected_s3_event)
def test_create_mediapackage_channel(self):
"""Create an AWS mediapackage channel."""
key = "video-key"
ssm_response = {"Version": 1, "Tier": "Standard"}
mediapackage_create_channel_response = {
"Id": f"test_{key}",
"HlsIngest": {
"IngestEndpoints": [
{
"Id": "ingest1",
"Password": "******",
"Url": "https://endpoint1/channel",
"Username": "******",
},
{
"Id": "ingest2",
"Password": "******",
"Url": "https://endpoint2/channel",
"Username": "******",
},
]
},
}
mediapackage_create_hls_origin_endpoint_response = {
"ChannelId": "channel1",
"Url": "https://endpoint1/channel.m3u8",
"Id": "enpoint1",
}
with Stubber(
medialive_utils.mediapackage_client
) as mediapackage_stubber, Stubber(medialive_utils.ssm_client) as ssm_stubber:
mediapackage_stubber.add_response(
"create_channel",
service_response=mediapackage_create_channel_response,
expected_params={
"Id": f"test_{key}",
"Tags": {"environment": "test", "app": "marsha"},
},
)
ssm_stubber.add_response(
"put_parameter",
service_response=ssm_response,
expected_params={
"Name": "test_user1",
"Description": "video-key MediaPackage Primary Ingest Username",
"Value": "password1",
"Type": "String",
"Tags": [
{"Key": "environment", "Value": "test"},
{"Key": "app", "Value": "marsha"},
],
},
)
ssm_stubber.add_response(
"put_parameter",
service_response=ssm_response,
expected_params={
"Name": "test_user2",
"Description": "video-key MediaPackage Secondary Ingest Username",
"Value": "password2",
"Type": "String",
"Tags": [
{"Key": "environment", "Value": "test"},
{"Key": "app", "Value": "marsha"},
],
},
)
mediapackage_stubber.add_response(
"create_origin_endpoint",
service_response=mediapackage_create_hls_origin_endpoint_response,
expected_params={
"ChannelId": f"test_{key}",
"Id": "test_video-key_hls",
"ManifestName": "test_video-key_hls",
"StartoverWindowSeconds": 86400,
"TimeDelaySeconds": 0,
"HlsPackage": {
"AdMarkers": "PASSTHROUGH",
"IncludeIframeOnlyStream": False,
"PlaylistType": "EVENT",
"PlaylistWindowSeconds": 1,
"ProgramDateTimeIntervalSeconds": 0,
"SegmentDurationSeconds": 5,
},
"Tags": {"environment": "test", "app": "marsha"},
},
)
[
channel,
hls_endpoint,
] = medialive_utils.create_mediapackage_channel(key)
mediapackage_stubber.assert_no_pending_responses()
ssm_stubber.assert_no_pending_responses()
self.assertEqual(channel, mediapackage_create_channel_response)
self.assertEqual(hls_endpoint, mediapackage_create_hls_origin_endpoint_response)
