def test_environment_account_mapping_with_default(self, fixtures):
accounts = fixtures['accounts']
dev_account = accounts[0]['alias']
live_account = accounts[-1]['alias']
live_environment = fixtures['environments'][0]
environments = {
live_environment: live_account,
'*': dev_account,
}
raw_scheme = {
'accounts':
{a['alias']: {
'id': a['id'],
'role': a['role']
}
for a in accounts},
'release-account': dev_account,
'release-bucket': 'releases',
'default-region': 'eu-west-69',
'environments': environments,
'terraform-backend-s3-bucket': 'tfstate-bucket',
'terraform-backend-s3-dynamodb-table': 'tflocks-table',
}
account_scheme = AccountScheme.create(raw_scheme, 'a-team')
assert account_scheme.account_for_environment(live_environment).alias \
== live_account
for environment in fixtures['environments'][1:]:
assert account_scheme.account_for_environment(environment).alias \
== dev_account
def test_account_with_region_override(self):
raw_scheme = {
'accounts': {
'prod': {
'id': '0987654321',
'role': 'admin-role',
},
'release': {
'id': '1234567890',
'role': 'test-role',
'region': 'region-override',
},
},
'environments': {},
'release-account': 'release',
'release-bucket': 'release-bucket',
'default-region': 'test-region-1',
'terraform-backend-s3-bucket': 'backend-bucket',
'terraform-backend-s3-dynamodb-table': 'backend-table',
'lambda-buckets': {
'test-region-1': 'test-bucket-1',
'test-region-2': 'test-bucket-2'
}
}
account_scheme = AccountScheme.create(raw_scheme, 'test-team')
assert len(account_scheme.accounts) == 2
for account in account_scheme.accounts:
if account.alias == 'prod':
assert account.region == 'test-region-1'
if account.alias == 'release':
assert account.region == 'region-override'
def test_substitute_team(self, team):
raw_scheme = {
'accounts': {
'{team}-release-account-{team}': {
'id': '123456789',
'role': '{team}-role-{team}',
}
},
'release-bucket': '{team}-release-bucket-{team}',
'lambda-bucket': '{team}-lambda-bucket-{team}',
'release-account': '{team}-release-account-{team}',
'default-region': '{team}-region-{team}',
'environments': {},
'terraform-backend-s3-bucket': '{team}-backend-bucket-{team}',
'terraform-backend-s3-dynamodb-table':
'{team}-backend-dynamo-{team}',
}
account_scheme = AccountScheme.create(raw_scheme, team)
assert account_scheme.release_account.id == '123456789'
assert account_scheme.release_account.role == \
'{team}-role-{team}'.format(team=team)
assert account_scheme.default_region == \
'{team}-region-{team}'.format(team=team)
assert account_scheme.accounts == {account_scheme.release_account}
assert account_scheme.release_bucket == \
'{team}-release-bucket-{team}'.format(team=team)
assert account_scheme.lambda_bucket == \
'{team}-lambda-bucket-{team}'.format(team=team)
assert account_scheme.backend_s3_bucket == \
'{team}-backend-bucket-{team}'.format(team=team)
assert account_scheme.backend_s3_dynamodb_table == \
'{team}-backend-dynamo-{team}'.format(team=team)
def test_lifecycle_policy_gets_enforced(self, fixtures):
component_name = fixtures['component_name']
accounts = fixtures['accounts']
account_ids = [account['id'] for account in accounts]
assume(len(set(account_ids)) == len(account_ids))
self._release.component_name = component_name
account_scheme = AccountScheme.create(
{
'accounts': {
account['alias']: {
'id': account['id'],
'role': account['role']
}
for account in accounts
},
'release-account': accounts[0]['alias'],
'default-region': self._region,
'release-bucket': 'dummy',
'environments': {
'live': accounts[0]['alias'],
},
'terraform-backend-s3-bucket': 'tfstate-bucket',
'terraform-backend-s3-dynamodb-table': 'tflocks-table',
}, 'a-team')
plugin = ReleasePlugin(self._release, account_scheme)
self._ecr_client.get_lifecycle_policy = Mock()
self._ecr_client.get_lifecycle_policy.side_effect = ClientError(
{'Error': {
'Code': 'LifecyclePolicyNotFoundException'
}}, None)
self._ecr_client.put_lifecycle_policy = Mock()
with patch('cdflow_commands.plugins.ecs.check_call'):
plugin.create()
self._ecr_client.put_lifecycle_policy.assert_called_once_with(
registryId=accounts[0]['id'],
repositoryName=component_name,
lifecyclePolicyText=json.dumps({
"rules": [{
"rulePriority": 1,
"description":
"Keep 500 tagged images (we tag all images), expire all others", # noqa
"selection": {
"tagStatus":
"tagged",
"tagPrefixList":
["1", "2", "3", "4", "5", "6", "7", "8",
"9"], # noqa
"countType":
"imageCountMoreThan",
"countNumber":
500
},
"action": {
"type": "expire"
}
}]
}))
def test_does_not_migrate_state_if_already_migrated(self):
with freeze_time("2012-01-14") as frozen_time:
account_scheme = AccountScheme.create(self.raw_scheme, self.team)
old_scheme = AccountScheme.create(self.old_raw_scheme, self.team)
environments = ('ci', 'qa', 'aslive', 'live')
state_written_time = deepcopy(frozen_time)
for env in environments:
self.s3_resource.Object(
self.old_state_bucket,
f'{env}/{self.component_name}/terraform.tfstate',
).put(Body=f'{env} state'.encode('utf-8'))
self.s3_resource.Object(
self.new_state_bucket,
(f'{self.team}/{self.component_name}/'
f'{env}/terraform.tfstate'),
).put(Body=f'{env} state'.encode('utf-8'))
self.s3_resource.Object(
self.new_state_bucket,
f'{self.team}/{self.component_name}/{env}/MIGRATED',
).put(Body=b'1')
frozen_time.tick(datetime.timedelta(days=1))
root_session = Session(region_name='eu-west-1')
migrate_state(
root_session,
account_scheme,
old_scheme,
self.team,
self.component_name,
)
for env in environments:
new_state = self.s3_resource.Object(
self.new_state_bucket,
(f'{self.team}/{self.component_name}/'
f'{env}/terraform.tfstate'),
)
assert new_state.last_modified.strftime("%Y-%m-%d") == \
state_written_time.time_to_freeze.strftime("%Y-%m-%d")
def test_migrate_component_not_deployed_to_all_environments(self):
account_scheme = AccountScheme.create(self.raw_scheme, self.team)
old_scheme = AccountScheme.create(self.old_raw_scheme, self.team)
environments = ('aslive', 'live')
for env in environments:
self.s3_resource.Object(
self.old_state_bucket,
f'{env}/{self.component_name}/terraform.tfstate',
).put(Body=f'{env} state'.encode('utf-8'))
for env in ('ci', 'qa', 'acceptance'):
self.s3_resource.Object(
self.old_state_bucket,
f'{env}/other-component/terraform.tfstate',
).put(Body=f'{env} state'.encode('utf-8'))
root_session = Session(region_name='eu-west-1')
migrate_state(
root_session,
account_scheme,
old_scheme,
self.team,
self.component_name,
)
for env in environments:
new_state = self.s3_resource.Object(
self.new_state_bucket,
f'{self.team}/{self.component_name}/{env}/terraform.tfstate',
)
response = new_state.get()
body = response['Body'].read()
assert body == f'{env} state'.encode()
migrated_marker = self.s3_resource.Object(
self.new_state_bucket,
f'{self.team}/{self.component_name}/{env}/MIGRATED',
)
migrated_response = migrated_marker.get()
migrated_body = migrated_response['Body'].read()
assert migrated_body == b'1'
def test_migrate_with_single_environment(self):
del self.raw_scheme['accounts']['prod']
del self.old_raw_scheme['accounts']['prod']
account_scheme = AccountScheme.create(self.raw_scheme, self.team)
old_scheme = AccountScheme.create(self.old_raw_scheme, self.team)
original_state = self.s3_resource.Object(
self.old_state_bucket,
f'test/{self.component_name}/terraform.tfstate',
)
original_state.put(Body=b'state')
root_session = Session(region_name='eu-west-1')
migrate_state(
root_session,
account_scheme,
old_scheme,
self.team,
self.component_name,
)
new_state = self.s3_resource.Object(
self.new_state_bucket,
f'{self.team}/{self.component_name}/test/terraform.tfstate',
)
response = new_state.get()
body = response['Body'].read()
assert body == b'state'
migrated_marker = self.s3_resource.Object(
self.new_state_bucket,
f'{self.team}/{self.component_name}/test/MIGRATED',
)
migrated_response = migrated_marker.get()
migrated_body = migrated_response['Body'].read()
assert migrated_body == b'1'
def build_account_scheme_s3(s3_resource, s3_url, team, component_name):
bucket, key = parse_s3_url(s3_url)
account_scheme = AccountScheme.create(
fetch_account_scheme(s3_resource, bucket, key), team)
upgrade = account_scheme.raw_scheme.get('upgrade-account-scheme')
def whitelisted(team, component_name):
team_whitelist = upgrade.get('team-whitelist', [])
component_whitelist = upgrade.get('component-whitelist', [])
return team in team_whitelist or component_name in component_whitelist
if upgrade and whitelisted(team, component_name):
old_scheme = account_scheme
new_s3_url = upgrade['new-url']
new_bucket, new_key = parse_s3_url(new_s3_url)
account_scheme = AccountScheme.create(
fetch_account_scheme(s3_resource, new_bucket, new_key), team)
logger.warning('Account scheme is being upgraded. Manually update '
f'account_scheme_url in cdflow.yml to {new_s3_url}')
else:
old_scheme = None
return account_scheme, old_scheme
def test_builds_container_and_uses_release_account_region(self, fixtures):
# Given
component_name = fixtures['component_name']
region = fixtures['region']
account_id = fixtures['account_id']
release = self._release
release.component_name = component_name
release.version = None
account_scheme = AccountScheme.create(
{
'accounts': {
'dummy': {
'id': account_id,
'role': 'dummy',
'region': region,
}
},
'release-account': 'dummy',
'default-region': 'eu-west-42',
'release-bucket': 'dummy',
'environments': {
'live': 'dummy',
},
'terraform-backend-s3-bucket': 'tfstate-bucket',
'terraform-backend-s3-dynamodb-table': 'tflocks-table',
}, 'a-team')
image_name = '{}.dkr.ecr.{}.amazonaws.com/{}:{}'.format(
account_id, region, component_name, 'dev')
plugin = ReleasePlugin(release, account_scheme)
with patch('cdflow_commands.plugins.ecs.check_call') as check_call:
# When
plugin_data = plugin.create()
# Then
assert plugin_data == {'image_id': image_name}
check_call.assert_called_once_with(
['docker', 'build', '-t', image_name, '.'])
def setUp(self):
boto_session = Mock()
self._ecr_client = Mock()
boto_session.client.return_value = self._ecr_client
self._release = MagicMock(spec=Release)
self._release.multi_region = False
self._release._team = 'a-team'
self._release.boto_session = boto_session
self._component_name = 'dummy-component'
self._source_dir = 'src'
self._release.component_name = self._component_name
self._version = '1.2.3'
self._release.version = self._version
self._region = 'dummy-region'
self._account_id = 'dummy-account-id'
self._account_scheme = AccountScheme.create(
{
'accounts': {
'dummy': {
'id': self._account_id,
'role': 'dummy'
}
},
'release-account': 'dummy',
'default-region': self._region,
'release-bucket': 'dummy',
'lambda-bucket': 'dummy-lambda-bucket',
'lambda-buckets': {
'test-region1': 'dummy-lambda-bucket',
'test-region2': 'dummy-lambda-bucket2',
},
'environments': {
'live': 'dummy',
},
'terraform-backend-s3-bucket': 'tfstate-bucket',
'terraform-backend-s3-dynamodb-table': 'tflocks-table',
}, 'a-team')
self._plugin = ReleasePlugin(self._release, self._account_scheme)
def test_multi_region_lambdas(self):
raw_scheme = {
'accounts': {
'release': {
'id': '1234567890',
'role': 'test-role'
}
},
'environments': {},
'release-account': 'release',
'release-bucket': 'release-bucket',
'default-region': 'test-region-1',
'terraform-backend-s3-bucket': 'backend-bucket',
'terraform-backend-s3-dynamodb-table': 'backend-table',
'lambda-buckets': {
'test-region-1': 'test-bucket-1',
'test-region-2': 'test-bucket-2'
}
}
account_scheme = AccountScheme.create(raw_scheme, 'test-team')
assert account_scheme.lambda_buckets == raw_scheme['lambda-buckets']
def test_create_account_scheme_from_json(self, fixtures):
raw_scheme = {
'accounts': {
fixtures['account']['alias']: {
'id': fixtures['account']['id'],
'role': fixtures['account']['role'],
}
},
'release-bucket':
fixtures['release-bucket'],
'lambda-bucket':
fixtures['lambda-bucket'],
'classic-metadata-handling':
fixtures['classic-metadata-handling'],
'release-account':
fixtures['account']['alias'],
'default-region':
fixtures['region'],
'environments': {},
'terraform-backend-s3-bucket':
fixtures['backend-s3-bucket'],
'terraform-backend-s3-dynamodb-table':
fixtures['backend-s3-dynamodb-table'],
}
account_scheme = AccountScheme.create(raw_scheme, 'a-team')
assert account_scheme.release_account.id == fixtures['account']['id']
assert account_scheme.release_account.role == \
fixtures['account']['role']
assert account_scheme.default_region == fixtures['region']
assert account_scheme.accounts == {account_scheme.release_account}
assert account_scheme.release_bucket == fixtures['release-bucket']
assert account_scheme.lambda_bucket == fixtures['lambda-bucket']
assert account_scheme.classic_metadata_handling == \
fixtures['classic-metadata-handling']
assert account_scheme.backend_s3_bucket == \
fixtures['backend-s3-bucket']
assert account_scheme.backend_s3_dynamodb_table == \
fixtures['backend-s3-dynamodb-table']
def test_deploy_account_ids(self, accounts):
raw_scheme = {
'accounts':
{a['alias']: {
'id': a['id'],
'role': a['role']
}
for a in accounts},
'release-account': accounts[0]['alias'],
'release-bucket': 'releases',
'default-region': 'eu-west-69',
'environments': {},
'terraform-backend-s3-bucket': 'tfstate-bucket',
'terraform-backend-s3-dynamodb-table': 'tflocks-table',
}
account_scheme = AccountScheme.create(raw_scheme, 'a-team')
expected_account_ids = list(
sorted(account['id'] for account in accounts))
assert list(sorted(account_scheme.account_ids)) == expected_account_ids
def setUp(self):
boto_session = Mock()
self._ecr_client = Mock()
boto_session.client.return_value = self._ecr_client
self._set_mock_get_authorization_token()
self._release = MagicMock(spec=Release)
self._release.boto_session = boto_session
self._component_name = 'dummy-component'
self._release.component_name = self._component_name
self._version = '1.2.3'
self._release.version = self._version
self._region = 'dummy-region'
self._account_id = 'dummy-account-id'
account_scheme = AccountScheme.create(
{
'accounts': {
'dummy': {
'id': self._account_id,
'role': 'dummy'
}
},
'classic-metadata-handling': True,
'release-account': 'dummy',
'default-region': self._region,
'release-bucket': 'dummy',
'environments': {
'live': 'dummy'
},
'terraform-backend-s3-bucket': 'tfstate-bucket',
'terraform-backend-s3-dynamodb-table': 'tflocks-table'
}, 'a-team')
self._plugin = ReleasePlugin(self._release, account_scheme)
def build_account_scheme_file(filename, team):
with open(filename) as f:
return AccountScheme.create(json.loads(f.read()), team)
def test_policy_set_on_repo(self, fixtures):
# unique_by above only supports a single hashable type, so we have to
# ensure uniqueness of account ids here - this is needed because the
# under test should only creates one statement for each unique account
# id
component_name = fixtures['component_name']
accounts = fixtures['accounts']
account_ids = [account['id'] for account in accounts]
assume(len(set(account_ids)) == len(account_ids))
self._release.component_name = component_name
account_scheme = AccountScheme.create(
{
'accounts': {
account['alias']: {
'id': account['id'],
'role': account['role']
}
for account in accounts
},
'classic-metadata-handling': True,
'release-account': accounts[0]['alias'],
'default-region': self._region,
'release-bucket': 'dummy',
'environments': {
'live': accounts[0]['alias'],
},
'terraform-backend-s3-bucket': 'tfstate-bucket',
'terraform-backend-s3-dynamodb-table': 'tflocks-table',
}, 'a-team')
plugin = ReleasePlugin(self._release, account_scheme)
self._ecr_client.set_repository_policy = Mock()
with patch('cdflow_commands.plugins.ecs.check_call'):
plugin.create()
expected_account_ids = sorted(
[account['id'] for account in accounts[1:]])
self._ecr_client.set_repository_policy.assert_called_once_with(
repositoryName=component_name,
policyText=json.dumps(
{
'Version':
'2008-10-17',
'Statement': [{
'Sid':
'allow {}'.format(account_id),
'Effect':
'Allow',
'Principal': {
'AWS':
'arn:aws:iam::{}:root'.format(account_id)
},
'Action': [
'ecr:GetDownloadUrlForLayer',
'ecr:BatchGetImage',
'ecr:BatchCheckLayerAvailability'
]
} for account_id in expected_account_ids]
},
sort_keys=True))