def ensure_trigger_dynamodb(name, arn_lambda, metadata, preview): triggers = [] for trigger in metadata['trigger']: if trigger.split()[0] == 'dynamodb': kind, table_name, *attrs = trigger.split() triggers.append([table_name, attrs]) if triggers: stderr('\nensure triggers dynamodb:') for table_name, attrs in triggers: ensure_attrs = {k: int(v) if v.isdigit() else v for a in attrs for k, v in [a.split('=')]} for k, v in trigger_dynamodb_attr_shortcuts.items(): if k in ensure_attrs: ensure_attrs[v] = ensure_attrs.pop(k) if 'StartingPosition' in ensure_attrs: ensure_attrs['StartingPosition'] = ensure_attrs['StartingPosition'].upper() if preview: stderr(' preview:', table_name) else: stream_arn = aws.dynamodb.stream_arn(table_name) conflict = client('lambda').exceptions.ResourceConflictException try: retry(client('lambda').create_event_source_mapping, conflict)(EventSourceArn=stream_arn, FunctionName=name, Enabled=True, **ensure_attrs) stderr('', table_name) except conflict as e: *_, kind, uuid = e.args[0].split() resp = client('lambda').get_event_source_mapping(UUID=uuid) for k, v in ensure_attrs.items(): if k != 'StartingPosition': assert resp[k] == v, [resp[k], v] stderr('', table_name)
def ensure_trigger_cloudwatch(name, arn_lambda, metadata, preview): triggers = [] for trigger in metadata['trigger']: if trigger.split()[0] == 'cloudwatch': kind, schedule = trigger.split(None, 1) triggers.append(schedule) if triggers: stderr('\nensure triggers cloudwatch:') assert len(triggers) == 1, f'only 1 cloudwatch schedule is currently supported: {triggers}' for schedule in triggers: if preview: stderr(' preview:', schedule) else: arn_rule = client('events').put_rule(Name=name, ScheduleExpression=schedule)['RuleArn'] ensure_permission(name, 'events.amazonaws.com', arn_rule) targets = retry(client('events').list_targets_by_rule)(Rule=name)['Targets'] assert all(t['Arn'] == arn_lambda for t in targets), f'there are unknown targets in cloudwatch rule: {name}' if len(targets) == 0: stderr('', schedule) client('events').put_targets(Rule=name, Targets=[{'Id': '1', 'Arn': arn_lambda}]) elif len(targets) == 1: assert targets[0]['Arn'] == arn_lambda, f'cloudwatch target mismatch: {arn_lambda} {targets[0]}' stderr('', schedule) elif len(targets) > 1: stderr(' removing:', schedule) targets = sorted(targets, key=lambda x: x['Id']) client('events').remove_targets(Rule=name, Ids=[t['Id'] for t in targets[1:]]) def ensure_only_one_target(): targets = client('events').list_targets_by_rule(Rule=name)['Targets'] assert len(targets) == 1, f'more than one target found for cloudwatch rule: {name} {schedule} {targets}' retry(ensure_only_one_target)()
def ls(selectors, state): assert state in ['running', 'pending', 'stopped', 'terminated', None], f'bad state: {state}' if not selectors: instances = list( retry(resource('ec2').instances.filter)(Filters=[{ 'Name': 'instance-state-name', 'Values': [state] }] if state else [])) else: kind = 'tags' kind = 'dns-name' if selectors[0].endswith('.amazonaws.com') else kind kind = 'vpc-id' if selectors[0].startswith('vpc-') else kind kind = 'subnet-id' if selectors[0].startswith('subnet-') else kind kind = 'instance.group-id' if selectors[0].startswith('sg-') else kind kind = 'private-dns-name' if selectors[0].endswith( '.ec2.internal') else kind kind = 'ip-address' if all(x.isdigit() or x == '.' for x in selectors[0]) else kind kind = 'private-ip-address' if all( x.isdigit() or x == '.' for x in selectors[0]) and selectors[0].split( '.')[0] in ['10', '172'] else kind kind = 'instance-id' if selectors[0].startswith('i-') else kind if kind == 'tags' and '=' not in selectors[0]: # TODO allow passing multiple names, which are summed, and then tags, which are intersected selectors = f'Name={selectors[0]}', *selectors[ 1:] # auto add Name= to the first tag instances = [] for chunk in util.iter.chunk(selectors, 195): # 200 boto api limit filters = [{ 'Name': 'instance-state-name', 'Values': [state] }] if state else [] try: if kind == 'tags': filters += [{ 'Name': f'tag:{k}', 'Values': [v] } for t in chunk for k, v in [t.split('=')]] else: filters += [{'Name': kind, 'Values': chunk}] except: pprint.pprint(selectors) pprint.pprint(filters) raise instances += list( retry(resource('ec2').instances.filter)(Filters=filters)) instances = sorted(instances, key=lambda i: i.instance_id) instances = sorted(instances, key=lambda i: tags(i).get('name', 'no-name')) instances = sorted(instances, key=lambda i: i.meta.data['LaunchTime'], reverse=True) return instances
def ensure_key_allows_role(arn_key, arn_role, preview): if not preview: resp = aws.client('kms').get_key_policy(KeyId=arn_key, PolicyName='default') policy = json.loads(resp['Policy']) # ensure that every Statement.Principal.AWS is a list, it can be either a # string or a list of strings. for statement in policy['Statement']: if statement.get('Principal', {}).get('AWS'): if isinstance(statement['Principal']['AWS'], str): statement['Principal']['AWS'] = [ statement['Principal']['AWS'] ] # remove invalid principals from all statements, these are caused by the # deletion of an iam role referenced by this policy, which transforms the # principal from something like "arn:..." to "AIEKFJ...". for statement in policy['Statement']: if statement.get('Principal', {}).get('AWS'): for arn in statement['Principal']['AWS'].copy(): if not arn.startswith('arn:'): statement['Principal']['AWS'].remove(arn) # ensure that the "allow use of key" Statement contains our role's arn for statement in policy['Statement']: if statement['Sid'] == 'Allow use of the key': if arn_role not in statement['Principal']['AWS']: statement['Principal']['AWS'].append(arn_role) break # if an "allow use of key" Statement didn't exist, create it else: policy['Statement'].append({ "Sid": "Allow use of the key", "Effect": "Allow", "Principal": { "AWS": [arn_role] }, "Action": [ "kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*", "kms:GenerateDataKey*", "kms:DescribeKey" ], "Resource": "*" }) try: retry(aws.client('kms').put_key_policy, silent=True)(KeyId=arn_key, Policy=json.dumps(policy), PolicyName='default') except aws.client('kms').exceptions.MalformedPolicyDocumentException: stderr(f'fatal: failed to put to key: {arn_key}, policy:\n' + json.dumps(policy, indent=2)) raise
def rm_table(name, print_fn=stderr): in_use = client('dynamodb').exceptions.ResourceInUseException not_found = client('dynamodb').exceptions.ResourceNotFoundException try: retry(client('dynamodb').delete_table, in_use, not_found)(TableName=name) except in_use as e: assert str(e).endswith(f'Table is being deleted: {name}'), e except not_found: pass else: print_fn('dynamodb deleted:', name)
def ensure_permission(name, principal, arn): not_found = client('lambda').exceptions.ResourceNotFoundException try: res = json.loads(retry(client('lambda').get_policy, not_found)(FunctionName=name)['Policy']) except not_found: statements = [] else: statements = [x['Sid'] for x in res['Statement']] id = principal.replace('.', '-') if id not in statements: client('lambda').add_permission(FunctionName=name, StatementId=id, Action='lambda:InvokeFunction', Principal=principal, SourceArn=arn)
def ensure_key(name, arn_user, arn_role, preview): stderr('\nensure kms key:') if preview: stderr(' preview: kms:', name) else: keys = [ x for x in all_keys() if x['AliasArn'].endswith(f':alias/lambda/{name}') ] if 0 == len(keys): arn_root = ':'.join(arn_user.split(':')[:-1]) + ':root' policy = """ {"Version": "2012-10-17", "Statement": [{"Sid": "Enable IAM User Permissions", "Effect": "Allow", "Principal": {"AWS": ["%(arn_user)s", "%(arn_root)s"]}, "Action": "kms:*", "Resource": "*"}, {"Sid": "Allow use of the key", "Effect": "Allow", "Principal": {"AWS": ["%(arn_user)s", "%(arn_role)s", "%(arn_root)s"]}, "Action": ["kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*", "kms:GenerateDataKey*", "kms:DescribeKey"], "Resource": "*"}, {"Sid": "Allow attachment of persistent resources", "Effect": "Allow", "Principal": {"AWS": ["%(arn_user)s", "%(arn_role)s", "%(arn_root)s"]}, "Action": ["kms:CreateGrant", "kms:ListGrants", "kms:RevokeGrant"], "Resource": "*", "Condition": {"Bool": {"kms:GrantIsForAWSResource": true}}}]} """ % { 'arn_role': arn_role, 'arn_user': arn_user, 'arn_root': arn_root } _key_id = retry(aws.client('kms').create_key, silent=True)( Policy=policy, Description=name)['KeyMetadata']['KeyId'] aws.client('kms').create_alias(AliasName=f'alias/lambda/{name}', TargetKeyId=_key_id) keys = [ x for x in all_keys() if x['AliasArn'].endswith(f':alias/lambda/{name}') ] assert len(keys) == 1 stderr('', keys[0]['AliasArn']) return key_id(keys[0]) elif 1 == len(keys): stderr('', keys[0]['AliasArn']) return key_id(keys[0]) else: stderr('fatal: found more than 1 key for:', name, '\n' + '\n'.join(keys)) sys.exit(1)
def ensure_table(name, *attrs, preview=False, yes=False, print_fn=stderr): # grab some exception shortcuts table_exists = client('dynamodb').exceptions.ResourceInUseException not_found = client('dynamodb').exceptions.ResourceNotFoundException client_error = client('dynamodb').exceptions.ClientError # start ensure_attrs with columns columns = [attr for attr in attrs if '=' not in attr] ensure_attrs = dicts.to_dotted({ 'AttributeDefinitions': [{ 'AttributeName': attr_name, 'AttributeType': attr_type.upper() } for column in columns for attr_name, attr_type, _ in [column.split(':')]], 'KeySchema': [{ 'AttributeName': attr_name, 'KeyType': key_type.upper() } for column in columns for attr_name, _, key_type in [column.split(':')]] }) # update ensure_attrs with the rest of the passed attributes ensure_attrs.update({ k: int(v) if v.isdigit() else v for attr in attrs if '=' in attr for k, v in [attr.split('=')] }) # resolve any attribute shortcuts for k, v in table_attr_shortcuts.items(): if k in ensure_attrs: ensure_attrs[v] = ensure_attrs.pop(k) # allow lower case for stream view type if 'StreamSpecification.StreamViewType' in ensure_attrs: ensure_attrs['StreamSpecification.StreamEnabled'] = True ensure_attrs['StreamSpecification.StreamViewType'] = ensure_attrs[ 'StreamSpecification.StreamViewType'].upper() # check provisioning and set billing type read = ensure_attrs.get('ProvisionedThroughput.ReadCapacityUnits') write = ensure_attrs.get('ProvisionedThroughput.WriteCapacityUnits') assert (not read and not write) or ( read and write ), 'both read and write must be provisioned, or neither for on-demand' ensure_attrs['BillingMode'] = 'PROVISIONED' if read else 'PAY_PER_REQUEST' # print and prompt print_fn() print_fn('TableName:', name) for k, v in ensure_attrs.items(): print_fn(f' {k}: {v}') print_fn() # fetch existing table attrs try: existing_attrs = dicts.to_dotted( client('dynamodb').describe_table(TableName=name)['Table']) # create table except not_found: # create if preview: print_fn(' preview: created:', name) else: if not yes: print_fn('\nproceed? y/n ') assert sh.getch() == 'y' retry(client('dynamodb').create_table, table_exists, client_error)(TableName=name, **dicts.from_dotted(ensure_attrs)) print_fn(' created:', name) # check and maybe update existing table else: if preview: print_fn(' preview: exists:', name) else: print_fn(' exists:', name) # join tags into existing attributes existing_attrs.update( dicts.to_dotted({ 'Tags': [ tag for page in retry( client('dynamodb').get_paginator( 'list_tags_of_resource').paginate)( ResourceArn=arn(name)) for tag in page['Tags'] ] })) # remap existing attributes to the same schema as table attributes existing_attrs = { k.replace('BillingModeSummary.', ''): v for k, v in existing_attrs.items() } # check every attribute needs_update = False for k, v in ensure_attrs.items(): if v != existing_attrs.get(k): needs_update = True if preview: print_fn(f' preview: {k}: {existing_attrs.get(k)} -> {v}') else: print_fn(f' {k}: {existing_attrs.get(k)} -> {v}') assert k.split( '.' )[0] != 'KeySchema', 'KeySchema cannot be updated on existing tables' # collect tags to remove tags_to_remove = [] for tag in dicts.from_dotted(existing_attrs).get('Tags', []): if tag['Key'] not in [ t['Key'] for t in dicts.from_dotted(ensure_attrs).get('Tags', []) ]: tags_to_remove.append(tag['Key']) if preview: print_fn(f' preview: untag: {tag["Key"]}') else: print_fn(f' untag: {tag["Key"]}') if not preview: # prompt if updates if (needs_update or tags_to_remove) and not yes: print_fn('\nproceed? y/n ') assert sh.getch() == 'y' # update if needed if needs_update: # update tags ensure_attrs = dicts.from_dotted(ensure_attrs) if 'Tags' in ensure_attrs: for tag in ensure_attrs['Tags']: tag['Value'] = str(tag['Value']) client('dynamodb').tag_resource(ResourceArn=arn(name), Tags=ensure_attrs['Tags']) del ensure_attrs['Tags'] # update table. note: KeySchema cannot be updated del ensure_attrs['KeySchema'] client('dynamodb').update_table(TableName=name, **ensure_attrs) # untag if needed if tags_to_remove: # remove unused tags if tags_to_remove: client('dynamodb').untag_resource(ResourceArn=arn(name), TagKeys=tags_to_remove)
def stream_arn(name): not_found = client('dynamodb').exceptions.ResourceNotFoundException return retry(client('dynamodb').describe_table, not_found)(TableName=name)['Table']['LatestStreamArn']
def arn(name): not_found = client('lambda').exceptions.ResourceNotFoundException return retry(client('lambda').get_function, not_found)(FunctionName=name)['Configuration']['FunctionArn']