def ensure_trigger_dynamodb(name, arn_lambda, metadata, preview):
triggers = []
for trigger in metadata['trigger']:
if trigger.split()[0] == 'dynamodb':
kind, table_name, *attrs = trigger.split()
triggers.append([table_name, attrs])
if triggers:
stderr('\nensure triggers dynamodb:')
for table_name, attrs in triggers:
ensure_attrs = {k: int(v) if v.isdigit() else v for a in attrs for k, v in [a.split('=')]}
for k, v in trigger_dynamodb_attr_shortcuts.items():
if k in ensure_attrs:
ensure_attrs[v] = ensure_attrs.pop(k)
if 'StartingPosition' in ensure_attrs:
ensure_attrs['StartingPosition'] = ensure_attrs['StartingPosition'].upper()
if preview:
stderr(' preview:', table_name)
else:
stream_arn = aws.dynamodb.stream_arn(table_name)
conflict = client('lambda').exceptions.ResourceConflictException
try:
retry(client('lambda').create_event_source_mapping, conflict)(EventSourceArn=stream_arn, FunctionName=name, Enabled=True, **ensure_attrs)
stderr('', table_name)
except conflict as e:
*_, kind, uuid = e.args[0].split()
resp = client('lambda').get_event_source_mapping(UUID=uuid)
for k, v in ensure_attrs.items():
if k != 'StartingPosition':
assert resp[k] == v, [resp[k], v]
stderr('', table_name)
def ensure_trigger_cloudwatch(name, arn_lambda, metadata, preview):
triggers = []
for trigger in metadata['trigger']:
if trigger.split()[0] == 'cloudwatch':
kind, schedule = trigger.split(None, 1)
triggers.append(schedule)
if triggers:
stderr('\nensure triggers cloudwatch:')
assert len(triggers) == 1, f'only 1 cloudwatch schedule is currently supported: {triggers}'
for schedule in triggers:
if preview:
stderr(' preview:', schedule)
else:
arn_rule = client('events').put_rule(Name=name, ScheduleExpression=schedule)['RuleArn']
ensure_permission(name, 'events.amazonaws.com', arn_rule)
targets = retry(client('events').list_targets_by_rule)(Rule=name)['Targets']
assert all(t['Arn'] == arn_lambda for t in targets), f'there are unknown targets in cloudwatch rule: {name}'
if len(targets) == 0:
stderr('', schedule)
client('events').put_targets(Rule=name, Targets=[{'Id': '1', 'Arn': arn_lambda}])
elif len(targets) == 1:
assert targets[0]['Arn'] == arn_lambda, f'cloudwatch target mismatch: {arn_lambda} {targets[0]}'
stderr('', schedule)
elif len(targets) > 1:
stderr(' removing:', schedule)
targets = sorted(targets, key=lambda x: x['Id'])
client('events').remove_targets(Rule=name, Ids=[t['Id'] for t in targets[1:]])
def ensure_only_one_target():
targets = client('events').list_targets_by_rule(Rule=name)['Targets']
assert len(targets) == 1, f'more than one target found for cloudwatch rule: {name} {schedule} {targets}'
retry(ensure_only_one_target)()
def ls(selectors, state):
assert state in ['running', 'pending', 'stopped', 'terminated',
None], f'bad state: {state}'
if not selectors:
instances = list(
retry(resource('ec2').instances.filter)(Filters=[{
'Name': 'instance-state-name',
'Values': [state]
}] if state else []))
else:
kind = 'tags'
kind = 'dns-name' if selectors[0].endswith('.amazonaws.com') else kind
kind = 'vpc-id' if selectors[0].startswith('vpc-') else kind
kind = 'subnet-id' if selectors[0].startswith('subnet-') else kind
kind = 'instance.group-id' if selectors[0].startswith('sg-') else kind
kind = 'private-dns-name' if selectors[0].endswith(
'.ec2.internal') else kind
kind = 'ip-address' if all(x.isdigit() or x == '.'
for x in selectors[0]) else kind
kind = 'private-ip-address' if all(
x.isdigit() or x == '.'
for x in selectors[0]) and selectors[0].split(
'.')[0] in ['10', '172'] else kind
kind = 'instance-id' if selectors[0].startswith('i-') else kind
if kind == 'tags' and '=' not in selectors[0]:
# TODO allow passing multiple names, which are summed, and then tags, which are intersected
selectors = f'Name={selectors[0]}', *selectors[
1:] # auto add Name= to the first tag
instances = []
for chunk in util.iter.chunk(selectors, 195): # 200 boto api limit
filters = [{
'Name': 'instance-state-name',
'Values': [state]
}] if state else []
try:
if kind == 'tags':
filters += [{
'Name': f'tag:{k}',
'Values': [v]
} for t in chunk for k, v in [t.split('=')]]
else:
filters += [{'Name': kind, 'Values': chunk}]
except:
pprint.pprint(selectors)
pprint.pprint(filters)
raise
instances += list(
retry(resource('ec2').instances.filter)(Filters=filters))
instances = sorted(instances, key=lambda i: i.instance_id)
instances = sorted(instances, key=lambda i: tags(i).get('name', 'no-name'))
instances = sorted(instances,
key=lambda i: i.meta.data['LaunchTime'],
reverse=True)
return instances
def ensure_key_allows_role(arn_key, arn_role, preview):
if not preview:
resp = aws.client('kms').get_key_policy(KeyId=arn_key,
PolicyName='default')
policy = json.loads(resp['Policy'])
# ensure that every Statement.Principal.AWS is a list, it can be either a
# string or a list of strings.
for statement in policy['Statement']:
if statement.get('Principal', {}).get('AWS'):
if isinstance(statement['Principal']['AWS'], str):
statement['Principal']['AWS'] = [
statement['Principal']['AWS']
]
# remove invalid principals from all statements, these are caused by the
# deletion of an iam role referenced by this policy, which transforms the
# principal from something like "arn:..." to "AIEKFJ...".
for statement in policy['Statement']:
if statement.get('Principal', {}).get('AWS'):
for arn in statement['Principal']['AWS'].copy():
if not arn.startswith('arn:'):
statement['Principal']['AWS'].remove(arn)
# ensure that the "allow use of key" Statement contains our role's arn
for statement in policy['Statement']:
if statement['Sid'] == 'Allow use of the key':
if arn_role not in statement['Principal']['AWS']:
statement['Principal']['AWS'].append(arn_role)
break
# if an "allow use of key" Statement didn't exist, create it
else:
policy['Statement'].append({
"Sid":
"Allow use of the key",
"Effect":
"Allow",
"Principal": {
"AWS": [arn_role]
},
"Action": [
"kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*",
"kms:GenerateDataKey*", "kms:DescribeKey"
],
"Resource":
"*"
})
try:
retry(aws.client('kms').put_key_policy,
silent=True)(KeyId=arn_key,
Policy=json.dumps(policy),
PolicyName='default')
except aws.client('kms').exceptions.MalformedPolicyDocumentException:
stderr(f'fatal: failed to put to key: {arn_key}, policy:\n' +
json.dumps(policy, indent=2))
raise
def rm_table(name, print_fn=stderr):
in_use = client('dynamodb').exceptions.ResourceInUseException
not_found = client('dynamodb').exceptions.ResourceNotFoundException
try:
retry(client('dynamodb').delete_table, in_use,
not_found)(TableName=name)
except in_use as e:
assert str(e).endswith(f'Table is being deleted: {name}'), e
except not_found:
pass
else:
print_fn('dynamodb deleted:', name)
def ensure_permission(name, principal, arn):
not_found = client('lambda').exceptions.ResourceNotFoundException
try:
res = json.loads(retry(client('lambda').get_policy, not_found)(FunctionName=name)['Policy'])
except not_found:
statements = []
else:
statements = [x['Sid'] for x in res['Statement']]
id = principal.replace('.', '-')
if id not in statements:
client('lambda').add_permission(FunctionName=name, StatementId=id, Action='lambda:InvokeFunction', Principal=principal, SourceArn=arn)
def ensure_key(name, arn_user, arn_role, preview):
stderr('\nensure kms key:')
if preview:
stderr(' preview: kms:', name)
else:
keys = [
x for x in all_keys()
if x['AliasArn'].endswith(f':alias/lambda/{name}')
]
if 0 == len(keys):
arn_root = ':'.join(arn_user.split(':')[:-1]) + ':root'
policy = """
{"Version": "2012-10-17",
"Statement": [{"Sid": "Enable IAM User Permissions",
"Effect": "Allow",
"Principal": {"AWS": ["%(arn_user)s", "%(arn_root)s"]},
"Action": "kms:*",
"Resource": "*"},
{"Sid": "Allow use of the key",
"Effect": "Allow",
"Principal": {"AWS": ["%(arn_user)s", "%(arn_role)s", "%(arn_root)s"]},
"Action": ["kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*", "kms:GenerateDataKey*", "kms:DescribeKey"],
"Resource": "*"},
{"Sid": "Allow attachment of persistent resources",
"Effect": "Allow",
"Principal": {"AWS": ["%(arn_user)s", "%(arn_role)s", "%(arn_root)s"]},
"Action": ["kms:CreateGrant", "kms:ListGrants", "kms:RevokeGrant"],
"Resource": "*",
"Condition": {"Bool": {"kms:GrantIsForAWSResource": true}}}]}
""" % {
'arn_role': arn_role,
'arn_user': arn_user,
'arn_root': arn_root
}
_key_id = retry(aws.client('kms').create_key, silent=True)(
Policy=policy, Description=name)['KeyMetadata']['KeyId']
aws.client('kms').create_alias(AliasName=f'alias/lambda/{name}',
TargetKeyId=_key_id)
keys = [
x for x in all_keys()
if x['AliasArn'].endswith(f':alias/lambda/{name}')
]
assert len(keys) == 1
stderr('', keys[0]['AliasArn'])
return key_id(keys[0])
elif 1 == len(keys):
stderr('', keys[0]['AliasArn'])
return key_id(keys[0])
else:
stderr('fatal: found more than 1 key for:', name,
'\n' + '\n'.join(keys))
sys.exit(1)
def ensure_table(name, *attrs, preview=False, yes=False, print_fn=stderr):
# grab some exception shortcuts
table_exists = client('dynamodb').exceptions.ResourceInUseException
not_found = client('dynamodb').exceptions.ResourceNotFoundException
client_error = client('dynamodb').exceptions.ClientError
# start ensure_attrs with columns
columns = [attr for attr in attrs if '=' not in attr]
ensure_attrs = dicts.to_dotted({
'AttributeDefinitions':
[{
'AttributeName': attr_name,
'AttributeType': attr_type.upper()
} for column in columns
for attr_name, attr_type, _ in [column.split(':')]],
'KeySchema': [{
'AttributeName': attr_name,
'KeyType': key_type.upper()
} for column in columns
for attr_name, _, key_type in [column.split(':')]]
})
# update ensure_attrs with the rest of the passed attributes
ensure_attrs.update({
k: int(v) if v.isdigit() else v
for attr in attrs if '=' in attr for k, v in [attr.split('=')]
})
# resolve any attribute shortcuts
for k, v in table_attr_shortcuts.items():
if k in ensure_attrs:
ensure_attrs[v] = ensure_attrs.pop(k)
# allow lower case for stream view type
if 'StreamSpecification.StreamViewType' in ensure_attrs:
ensure_attrs['StreamSpecification.StreamEnabled'] = True
ensure_attrs['StreamSpecification.StreamViewType'] = ensure_attrs[
'StreamSpecification.StreamViewType'].upper()
# check provisioning and set billing type
read = ensure_attrs.get('ProvisionedThroughput.ReadCapacityUnits')
write = ensure_attrs.get('ProvisionedThroughput.WriteCapacityUnits')
assert (not read and not write) or (
read and write
), 'both read and write must be provisioned, or neither for on-demand'
ensure_attrs['BillingMode'] = 'PROVISIONED' if read else 'PAY_PER_REQUEST'
# print and prompt
print_fn()
print_fn('TableName:', name)
for k, v in ensure_attrs.items():
print_fn(f' {k}: {v}')
print_fn()
# fetch existing table attrs
try:
existing_attrs = dicts.to_dotted(
client('dynamodb').describe_table(TableName=name)['Table'])
# create table
except not_found:
# create
if preview:
print_fn(' preview: created:', name)
else:
if not yes:
print_fn('\nproceed? y/n ')
assert sh.getch() == 'y'
retry(client('dynamodb').create_table, table_exists,
client_error)(TableName=name,
**dicts.from_dotted(ensure_attrs))
print_fn(' created:', name)
# check and maybe update existing table
else:
if preview:
print_fn(' preview: exists:', name)
else:
print_fn(' exists:', name)
# join tags into existing attributes
existing_attrs.update(
dicts.to_dotted({
'Tags': [
tag for page in retry(
client('dynamodb').get_paginator(
'list_tags_of_resource').paginate)(
ResourceArn=arn(name)) for tag in page['Tags']
]
}))
# remap existing attributes to the same schema as table attributes
existing_attrs = {
k.replace('BillingModeSummary.', ''): v
for k, v in existing_attrs.items()
}
# check every attribute
needs_update = False
for k, v in ensure_attrs.items():
if v != existing_attrs.get(k):
needs_update = True
if preview:
print_fn(f' preview: {k}: {existing_attrs.get(k)} -> {v}')
else:
print_fn(f' {k}: {existing_attrs.get(k)} -> {v}')
assert k.split(
'.'
)[0] != 'KeySchema', 'KeySchema cannot be updated on existing tables'
# collect tags to remove
tags_to_remove = []
for tag in dicts.from_dotted(existing_attrs).get('Tags', []):
if tag['Key'] not in [
t['Key']
for t in dicts.from_dotted(ensure_attrs).get('Tags', [])
]:
tags_to_remove.append(tag['Key'])
if preview:
print_fn(f' preview: untag: {tag["Key"]}')
else:
print_fn(f' untag: {tag["Key"]}')
if not preview:
# prompt if updates
if (needs_update or tags_to_remove) and not yes:
print_fn('\nproceed? y/n ')
assert sh.getch() == 'y'
# update if needed
if needs_update:
# update tags
ensure_attrs = dicts.from_dotted(ensure_attrs)
if 'Tags' in ensure_attrs:
for tag in ensure_attrs['Tags']:
tag['Value'] = str(tag['Value'])
client('dynamodb').tag_resource(ResourceArn=arn(name),
Tags=ensure_attrs['Tags'])
del ensure_attrs['Tags']
# update table. note: KeySchema cannot be updated
del ensure_attrs['KeySchema']
client('dynamodb').update_table(TableName=name, **ensure_attrs)
# untag if needed
if tags_to_remove:
# remove unused tags
if tags_to_remove:
client('dynamodb').untag_resource(ResourceArn=arn(name),
TagKeys=tags_to_remove)
def stream_arn(name):
not_found = client('dynamodb').exceptions.ResourceNotFoundException
return retry(client('dynamodb').describe_table,
not_found)(TableName=name)['Table']['LatestStreamArn']
def arn(name):
not_found = client('lambda').exceptions.ResourceNotFoundException
return retry(client('lambda').get_function, not_found)(FunctionName=name)['Configuration']['FunctionArn']