def ensure_trigger_cloudwatch(name, arn_lambda, metadata, preview):
triggers = []
for trigger in metadata['trigger']:
if trigger.split()[0] == 'cloudwatch':
kind, schedule = trigger.split(None, 1)
triggers.append(schedule)
if triggers:
stderr('\nensure triggers cloudwatch:')
assert len(triggers) == 1, f'only 1 cloudwatch schedule is currently supported: {triggers}'
for schedule in triggers:
if preview:
stderr(' preview:', schedule)
else:
arn_rule = client('events').put_rule(Name=name, ScheduleExpression=schedule)['RuleArn']
ensure_permission(name, 'events.amazonaws.com', arn_rule)
targets = retry(client('events').list_targets_by_rule)(Rule=name)['Targets']
assert all(t['Arn'] == arn_lambda for t in targets), f'there are unknown targets in cloudwatch rule: {name}'
if len(targets) == 0:
stderr('', schedule)
client('events').put_targets(Rule=name, Targets=[{'Id': '1', 'Arn': arn_lambda}])
elif len(targets) == 1:
assert targets[0]['Arn'] == arn_lambda, f'cloudwatch target mismatch: {arn_lambda} {targets[0]}'
stderr('', schedule)
elif len(targets) > 1:
stderr(' removing:', schedule)
targets = sorted(targets, key=lambda x: x['Id'])
client('events').remove_targets(Rule=name, Ids=[t['Id'] for t in targets[1:]])
def ensure_only_one_target():
targets = client('events').list_targets_by_rule(Rule=name)['Targets']
assert len(targets) == 1, f'more than one target found for cloudwatch rule: {name} {schedule} {targets}'
retry(ensure_only_one_target)()
def ensure_instance_profile_has_role(name, role_name, preview):
stderr('\nensure instance profile has role:')
profiles = [profile
for page in client('iam').get_paginator('list_instance_profiles').paginate()
for profile in page['InstanceProfiles']
if profile['InstanceProfileName'] == name]
if 0 == len(profiles):
if preview:
stderr(' preview: created:', name)
profile = None
else:
profile = client('iam').create_instance_profile(InstanceProfileName=name)['InstanceProfile']
stderr(' created:', name)
elif 1 == len(profiles):
if preview:
stderr(' preview: exists:', name)
else:
stderr(' exists:', name)
profile = profiles[0]
else:
assert False, profiles
if profile:
roles = [role['RoleName'] for role in profile['Roles']]
if role_name not in roles:
client('iam').add_role_to_instance_profile(InstanceProfileName=name, RoleName=role_name)
def set_concurrency(name, concurrency, preview):
if concurrency:
if preview:
stderr('\npreview: concurrency:', concurrency)
else:
client('lambda').put_function_concurrency(FunctionName=name, ReservedConcurrentExecutions=concurrency)
stderr('\nconcurrency:', concurrency)
def ensure_infra_s3(buckets, preview):
if buckets:
stderr('\nensure infra s3:')
for bucket in buckets:
name, *args = bucket.split()
assert not args, 'no params to s3'
aws.s3.ensure_bucket(name, print_fn=lambda *a: stderr('', *a), preview=preview)
def ensure_trigger_api(name, arn_lambda, metadata, preview):
for trigger in metadata['trigger']:
if trigger.split()[0] == 'api':
if preview:
stderr('\npreview: ensure triggers api')
else:
stderr('\nensure triggers api')
try:
rest_api_id = aws.api.api_id(name)
except AssertionError:
rest_api_id = client('apigateway').create_rest_api(name=name,
binaryMediaTypes=['*/*'],
endpointConfiguration={'types': ['REGIONAL']})['id']
parent_id = aws.api.resource_id(rest_api_id, '/')
resource_id = aws.api.resource_id(rest_api_id, '/{proxy+}')
if not resource_id:
resource_id = client('apigateway').create_resource(restApiId=rest_api_id, parentId=parent_id, pathPart='{proxy+}')['id']
api = client('lambda').meta.service_model.api_version
uri = f"arn:aws:apigateway:{aws.region()}:lambda:path/{api}/functions/"
uri += f'arn:aws:lambda:{aws.region()}:{aws.account()}:function:{name}/invocations'
for id in [parent_id, resource_id]:
try:
client('apigateway').put_method(restApiId=rest_api_id, resourceId=id, httpMethod='ANY', authorizationType='NONE')
except client('apigateway').exceptions.ConflictException:
pass
else:
client('apigateway').put_integration(restApiId=rest_api_id, resourceId=id, httpMethod='ANY', type="AWS_PROXY", integrationHttpMethod='POST', uri=uri)
client('apigateway').create_deployment(restApiId=rest_api_id, stageName=stage_name)
arn = f"arn:aws:execute-api:{aws.region()}:{aws.account()}:{rest_api_id}/*/*/*"
ensure_permission(name, 'apigateway.amazonaws.com', arn)
break
def update_zip(path, *includes):
stderr('\nupdate zip:')
_zip_file = zip_file(path)
tempdir = os.path.dirname(_zip_file)
[site_packages] = glob.glob(f'{tempdir}/env/lib/python3*/site-packages')
with sh.cd(site_packages):
sh.run(f'cp {path} .')
sh.run(f'zip {_zip_file} {os.path.basename(path)}')
stderr('', _zip_file)
def rm_instance_profile(name):
try:
client('iam').get_instance_profile(InstanceProfileName=name)
except client('iam').exceptions.NoSuchEntityException:
return
else:
for role in client('iam').get_instance_profile(InstanceProfileName=name)['InstanceProfile']['Roles']:
rm_role(role['RoleName'])
client('iam').delete_instance_profile(InstanceProfileName=name)
stderr(' deleted instance profile:', name)
def parse_file(path, silent=False):
if not os.path.isfile(path):
stderr('no such file:', path)
sys.exit(1)
if not path.endswith('.py') or len(path.split('.py')) > 2:
stderr('usage: python deploy.py some_lambda_file.py')
sys.exit(1)
with open(path) as f:
meta = metadata(f.read().splitlines(), silent=silent)
return path, meta
def ensure_key_allows_role(arn_key, arn_role, preview):
if not preview:
resp = aws.client('kms').get_key_policy(KeyId=arn_key,
PolicyName='default')
policy = json.loads(resp['Policy'])
# ensure that every Statement.Principal.AWS is a list, it can be either a
# string or a list of strings.
for statement in policy['Statement']:
if statement.get('Principal', {}).get('AWS'):
if isinstance(statement['Principal']['AWS'], str):
statement['Principal']['AWS'] = [
statement['Principal']['AWS']
]
# remove invalid principals from all statements, these are caused by the
# deletion of an iam role referenced by this policy, which transforms the
# principal from something like "arn:..." to "AIEKFJ...".
for statement in policy['Statement']:
if statement.get('Principal', {}).get('AWS'):
for arn in statement['Principal']['AWS'].copy():
if not arn.startswith('arn:'):
statement['Principal']['AWS'].remove(arn)
# ensure that the "allow use of key" Statement contains our role's arn
for statement in policy['Statement']:
if statement['Sid'] == 'Allow use of the key':
if arn_role not in statement['Principal']['AWS']:
statement['Principal']['AWS'].append(arn_role)
break
# if an "allow use of key" Statement didn't exist, create it
else:
policy['Statement'].append({
"Sid":
"Allow use of the key",
"Effect":
"Allow",
"Principal": {
"AWS": [arn_role]
},
"Action": [
"kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*",
"kms:GenerateDataKey*", "kms:DescribeKey"
],
"Resource":
"*"
})
try:
retry(aws.client('kms').put_key_policy,
silent=True)(KeyId=arn_key,
Policy=json.dumps(policy),
PolicyName='default')
except aws.client('kms').exceptions.MalformedPolicyDocumentException:
stderr(f'fatal: failed to put to key: {arn_key}, policy:\n' +
json.dumps(policy, indent=2))
raise
def ensure_key(name, arn_user, arn_role, preview):
stderr('\nensure kms key:')
if preview:
stderr(' preview: kms:', name)
else:
keys = [
x for x in all_keys()
if x['AliasArn'].endswith(f':alias/lambda/{name}')
]
if 0 == len(keys):
arn_root = ':'.join(arn_user.split(':')[:-1]) + ':root'
policy = """
{"Version": "2012-10-17",
"Statement": [{"Sid": "Enable IAM User Permissions",
"Effect": "Allow",
"Principal": {"AWS": ["%(arn_user)s", "%(arn_root)s"]},
"Action": "kms:*",
"Resource": "*"},
{"Sid": "Allow use of the key",
"Effect": "Allow",
"Principal": {"AWS": ["%(arn_user)s", "%(arn_role)s", "%(arn_root)s"]},
"Action": ["kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*", "kms:GenerateDataKey*", "kms:DescribeKey"],
"Resource": "*"},
{"Sid": "Allow attachment of persistent resources",
"Effect": "Allow",
"Principal": {"AWS": ["%(arn_user)s", "%(arn_role)s", "%(arn_root)s"]},
"Action": ["kms:CreateGrant", "kms:ListGrants", "kms:RevokeGrant"],
"Resource": "*",
"Condition": {"Bool": {"kms:GrantIsForAWSResource": true}}}]}
""" % {
'arn_role': arn_role,
'arn_user': arn_user,
'arn_root': arn_root
}
_key_id = retry(aws.client('kms').create_key, silent=True)(
Policy=policy, Description=name)['KeyMetadata']['KeyId']
aws.client('kms').create_alias(AliasName=f'alias/lambda/{name}',
TargetKeyId=_key_id)
keys = [
x for x in all_keys()
if x['AliasArn'].endswith(f':alias/lambda/{name}')
]
assert len(keys) == 1
stderr('', keys[0]['AliasArn'])
return key_id(keys[0])
elif 1 == len(keys):
stderr('', keys[0]['AliasArn'])
return key_id(keys[0])
else:
stderr('fatal: found more than 1 key for:', name,
'\n' + '\n'.join(keys))
sys.exit(1)
def ensure_trigger_sqs(name, arn_lambda, metadata, preview):
triggers = []
for trigger in metadata['trigger']:
if trigger.split()[0] == 'sqs':
kind, queue_name, *attrs = trigger.split()
triggers.append([queue_name, attrs])
if triggers:
stderr('\nensure triggers sqs:')
for queue_name, attrs in triggers:
ensure_attrs = {k: int(v) if v.isdigit() else v for a in attrs for k, v in [a.split('=')]}
for k, v in trigger_sqs_attr_shortcuts.items():
if k in ensure_attrs:
ensure_attrs[v] = ensure_attrs.pop(k)
if 'StartingPosition' in ensure_attrs:
ensure_attrs['StartingPosition'] = ensure_attrs['StartingPosition'].upper()
if preview:
stderr(' preview:', queue_name)
else:
stream_arn = aws.dynamodb.stream_arn(queue_name)
try:
client('lambda').create_event_source_mapping(EventSourceArn=stream_arn, FunctionName=name, Enabled=True, **ensure_attrs)
stderr('', queue_name)
except client('lambda').exceptions.ResourceConflictException as e:
*_, kind, uuid = e.args[0].split()
resp = client('lambda').get_event_source_mapping(UUID=uuid)
for k, v in ensure_attrs.items():
if k != 'StartingPosition':
assert resp[k] == v, [resp[k], v]
stderr('', queue_name)
def ensure_infra_log_group(name, preview):
name = f'/aws/lambda/{name}'
stderr('ensure infra logs:')
try:
if preview:
stderr(' preview:', name)
else:
client('logs').create_log_group(logGroupName=name)
stderr('', name)
except client('logs').exceptions.ResourceAlreadyExistsException:
stderr('', name)
def create_zip(path, requires, preview):
stderr('\ncreate zip:')
_zip_file = zip_file(path)
if not preview:
tempdir = os.path.dirname(_zip_file)
sh.run('rm -rf', tempdir)
sh.run('mkdir -p', tempdir)
sh.run(f'virtualenv --python python3 {tempdir}/env')
if requires:
for require in requires:
if preview:
stderr(' preview: require:', require)
else:
stderr(' require:', require)
if not preview:
with sh.cd(os.path.dirname(path)):
sh.run(f'{tempdir}/env/bin/pip install', *[f'"{r}"' for r in requires])
if not preview:
[site_packages] = glob.glob(f'{tempdir}/env/lib/python3*/site-packages')
with sh.cd(site_packages):
sh.run(f'cp {path} .')
sh.run('rm -rf wheel pip setuptools pkg_resources easy_install.py')
sh.run("ls | grep -E 'info$' | xargs rm -rf")
libs = sh.run('ls').splitlines()
for binpath in sh.run(f"find {tempdir}/env/bin/ -type f,l").splitlines():
name = os.path.basename(binpath)
if name not in libs:
with open(binpath, 'rb') as f:
lines = f.read().splitlines()
if lines and lines[0].startswith(b"#!") and b'python' in lines[0]:
with open(name, 'wb') as f:
f.write(b'#!/var/lang/bin/python\n')
f.write(b'\n'.join(lines[1:]))
sh.run('chmod +x', name)
sh.run(f'zip -r {_zip_file} .')
def rm_role(name):
try:
client('iam').get_role(RoleName=name)
except client('iam').exceptions.NoSuchEntityException:
return
else:
stderr(name)
policies = [policy for page in client('iam').get_paginator('list_attached_role_policies').paginate(RoleName=name) for policy in page['AttachedPolicies']]
for policy in policies:
client('iam').detach_role_policy(RoleName=name, PolicyArn=policy["PolicyArn"])
stderr(' detached policy:', policy["PolicyName"])
role_policies = [policy for page in client('iam').get_paginator('list_role_policies').paginate(RoleName=name) for policy in page['PolicyNames']]
for policy in role_policies:
client('iam').delete_role_policy(RoleName=name, PolicyName=policy)
stderr(' deleted policy:', policy)
profiles = [profile['InstanceProfileName']
for page in client('iam').get_paginator('list_instance_profiles_for_role').paginate(RoleName=name)
for profile in page['InstanceProfiles']]
for profile in profiles:
client('iam').remove_role_from_instance_profile(InstanceProfileName=profile, RoleName=name)
stderr(' detached from profile:', profile)
client('iam').delete_role(RoleName=name)
stderr(' deleted role:', name)
def include_in_zip(path, includes, preview):
_zip_file = zip_file(path)
with sh.cd(os.path.dirname(os.path.abspath(path))):
for include in includes:
if '*' in include:
for path in glob.glob(include):
if preview:
stderr(' preview: include:', path)
else:
stderr(' include:', path)
sh.run(f'zip {_zip_file} {path}')
else:
if preview:
stderr(' preview: include:', include)
else:
stderr(' include:', include)
sh.run(f'zip {_zip_file} {include}')
def ensure_infra_sns(snss, preview):
not_found = client('sns').exceptions.NotFoundException
if snss:
stderr('\nensure infra sns:')
for sns in snss:
name, *attrs = sns.split()
if preview:
stderr(' preview:', name)
else:
attrs = {k: int(v) if v.isdigit() else v for attr in attrs for k, v in [attr.split('=')]}
try:
sns_attrs = client('sns').get_topic_attributes(TopicArn=aws.sns.arn(name))
except not_found:
client('sns').create_topic(Name=sns)
stderr('', name)
else:
for k, v in attrs.items():
assert sns_attrs[k] == v, f'sns attr mismatch {k} {v} != {sns_attrs[k]}'
stderr('', name)
def ensure_infra_sqs(sqss, preview):
assert False, 'use dotted dict and move to aws'
not_found = client('sqs').exceptions.QueueDoesNotExist
if sqss:
stderr('\nensure infra sqs:')
for sqs in sqss:
name, *attrs = sqs.split()
if preview:
stderr(' preview:', name)
else:
attrs = {k: int(v) if v.isdigit() else v for attr in attrs for k, v in [attr.split('=')]}
try:
queue_url = client('sqs').get_queue_url(QueueName=sqs)
except not_found:
client('sqs').create_queue(QueueName=sqs, Attributes=attrs)
stderr('', name)
else:
queue_attrs = client('sqs').get_queue_attributes(QueueUrl=queue_url)['Attributes']
for k, v in attrs.items():
assert queue_attrs[k] == v, f'sqs attr mismatch {k} {v} != {queue_attrs[k]}'
stderr('', name)
def parse_metadata(token, lines, silent=False):
vals = [(line,
line.split(token, 1)[-1].split('#')[0].strip())
for line in filter_metadata(lines)
if line.startswith(token)]
new_vals = []
for line, val in vals:
if '$' in val:
try:
val = ''.join([os.environ[part[2:-1]] if part.startswith('$') else part for part in re.split('(\$\{[^\}]+})', val)])
except KeyError:
stderr(f'fatal: missing environment: {line}')
sys.exit(1)
new_vals.append(val)
vals = new_vals
if vals and not silent:
stderr(token)
for val in vals:
stderr('', val)
stderr()
return vals
def ensure_allows(name, allows, preview):
if allows:
stderr('\nensure allows:')
for allow in allows:
action, resource = allow.split()
if preview:
stderr(' preview:', allow)
else:
stderr('', allow)
policy = f'''{{"Version": "2012-10-17",
"Statement": [{{"Effect": "Allow",
"Action": "{action}",
"Resource": "{resource}"}}]}}'''
client('iam').put_role_policy(RoleName=name, PolicyName=_policy_name(allow), PolicyDocument=policy)
def ensure_trigger_sns(name, arn_lambda, metadata, preview):
triggers = []
for trigger in metadata['trigger']:
if trigger.split()[0] == 'sns':
kind, sns_name, *_ = trigger.split()
triggers.append(sns_name)
if triggers:
stderr('\nensure triggers sns:')
for sns in triggers:
if preview:
stderr(' preview:', sns_name)
else:
arn_sns = aws.sns.arn(sns_name)
subs = (sub for page in client('sns').get_paginator('list_subscriptions_by_topic').paginate(TopicArn=arn_sns) for sub in page['Subscriptions'])
for sub in subs:
if sub['Endpoint'] == arn_lambda:
stderr('', sns_name)
break
else:
client('sns').subscribe(TopicArn=arn_sns, Protocol='lambda', Endpoint=arn_lambda)
ensure_permission(name, 'sns.amazonaws.com', arn_sns)
stderr('', sns_name)
def ensure_trigger_s3(name, arn_lambda, metadata, preview):
events = ['s3:ObjectCreated:*', 's3:ObjectRemoved:*']
triggers = []
for trigger in metadata['trigger']:
if trigger.split()[0] == 's3':
kind, bucket, *_ = trigger.split()
triggers.append(bucket)
if triggers:
stderr('\nensure triggers s3:')
for bucket in triggers:
if preview:
stderr(' preview:', bucket)
else:
ensure_permission(name, 's3.amazonaws.com', f'arn:aws:s3:::{bucket}')
confs = aws.resource('s3').BucketNotification(bucket).lambda_function_configurations or []
for conf in confs:
if conf['LambdaFunctionArn'] == arn_lambda and conf['Events'] == events:
stderr('', bucket)
break
else:
confs.append({'LambdaFunctionArn': arn_lambda, 'Events': events})
stderr('', bucket)
aws.resource('s3').BucketNotification(bucket).put(NotificationConfiguration={'LambdaFunctionConfigurations': confs})
def rm_extra_allows(name, allows, preview):
to_remove = []
try:
role_policies = [policy for page in client('iam').get_paginator('list_role_policies').paginate(RoleName=name) for policy in page['PolicyNames']]
except client('iam').exceptions.NoSuchEntityException:
pass
else:
for policy in role_policies:
if policy not in [_policy_name(x) for x in allows]:
to_remove.append(policy)
if to_remove:
stderr('\nremove extra allows:')
for policy in to_remove:
if preview:
stderr(' preview:', policy)
else:
stderr('', policy)
client('iam').delete_role_policy(RoleName=name, PolicyName=policy)
def rm_extra_policies(name, policies, preview):
to_remove = []
try:
attached_role_policies = [policy for page in client('iam').get_paginator('list_attached_role_policies').paginate(RoleName=name) for policy in page['AttachedPolicies']]
except client('iam').exceptions.NoSuchEntityException:
pass
else:
for policy in attached_role_policies:
if policy['PolicyName'] not in policies:
to_remove.append(policy)
if to_remove:
stderr('\nremove extra policies:')
for policy in to_remove:
if preview:
stderr(' preview:', policy['PolicyName'])
else:
stderr('', policy['PolicyName'])
client('iam').detach_role_policy(RoleName=name, PolicyArn=policy["PolicyArn"])
def tail(group_name, follow=False, timestamps=False, exit_after=None):
stderr('group:', group_name)
if follow:
tokens = {}
limit = 3 # when starting to follow, dont page all history, just grab the last few entries and then start following
while True:
try:
stream_names = most_recent_streams(group_name)
except (IndexError,
aws.client('logs').exceptions.ResourceNotFoundException):
pass
else:
for stream_name in stream_names:
kw: Dict[str, Any] = {}
token = tokens.get(stream_name)
if token:
kw['nextToken'] = token
if limit != 0:
kw['limit'] = limit
limit = 0
resp = aws.client('logs').get_log_events(
logGroupName=group_name,
logStreamName=stream_name,
**kw)
if resp['events']:
tokens[stream_name] = resp['nextForwardToken']
for log in resp['events']:
if log['message'].split()[0] not in [
'START', 'END', 'REPORT'
]:
if timestamps:
print(datetime.datetime.fromtimestamp(
log['timestamp'] / 1000),
log['message'].replace('\t',
' ').strip(),
flush=True)
else:
print(log['message'].replace('\t',
' ').strip(),
flush=True)
if exit_after and exit_after in log['message']:
sys.exit(0)
time.sleep(1)
else:
try:
stream_names = most_recent_streams(group_name)
except IndexError:
stderr('no logs available')
sys.exit(1)
else:
for stream_name in stream_names:
stderr('group:', group_name, 'stream:', stream_name)
logs = aws.client('logs').get_log_events(
logGroupName=group_name,
logStreamName=stream_name)['events']
for log in logs:
if log['message'].split()[0] not in [
'START', 'END', 'REPORT'
]:
if timestamps:
print(datetime.datetime.fromtimestamp(
log['timestamp'] / 1000),
log['message'].replace('\t', ' ').strip(),
flush=True)
else:
print(log['message'].replace('\t', ' ').strip(),
flush=True)
def ensure_role(name, principal, preview):
stderr('\nensure role:')
if preview:
stderr(' preview:', name)
else:
role_path = f'/{principal}/{name}-path/'
roles = [role for page in client('iam').get_paginator('list_roles').paginate(PathPrefix=role_path) for role in page['Roles']]
if 0 == len(roles):
stderr('', name)
policy = '''{"Version": "2012-10-17",
"Statement": [{"Effect": "Allow",
"Principal": {"Service": "%s.amazonaws.com"},
"Action": "sts:AssumeRole"}]}''' % principal
client('iam').create_role(Path=role_path, RoleName=name, AssumeRolePolicyDocument=policy)
elif 1 == len(roles):
stderr('', name)
else:
stderr(' error: there is more than 1 role under path:', role_path)
for role in roles:
stderr('', role)
sys.exit(1)
def ensure_policies(name, policies, preview):
if policies:
stderr('\nensure policies:')
if preview:
all_policies = []
else:
all_policies = [policy for page in client('iam').get_paginator('list_policies').paginate() for policy in page['Policies']]
for policy in policies:
if preview:
stderr(' preview:', policy)
else:
matched_polices = [x for x in all_policies if x['Arn'].split('/')[-1] == policy]
if 0 == len(matched_polices):
stderr('fatal: didnt find any policy:', policy)
sys.exit(1)
elif 1 == len(matched_polices):
client('iam').attach_role_policy(RoleName=name, PolicyArn=matched_polices[0]["Arn"])
stderr('', policy)
else:
stderr('fatal: found more than 1 policy:', policy)
for p in matched_polices:
stderr(p['Arn'])
sys.exit(1)
def ensure_infra_dynamodb(dbs, preview):
if dbs:
stderr('\nensure infra dynamodb:')
for i, db in enumerate(dbs):
name, *args = db.split()
aws.dynamodb.ensure_table(name, *args, yes=True, print_fn=lambda *a: stderr('', *a), preview=preview)
