Esempio n. 1
0
def ensure_trigger_dynamodb(name, arn_lambda, metadata, preview):
    triggers = []
    for trigger in metadata['trigger']:
        if trigger.split()[0] == 'dynamodb':
            kind, table_name, *attrs = trigger.split()
            triggers.append([table_name, attrs])
    if triggers:
        stderr('\nensure triggers dynamodb:')
        for table_name, attrs in triggers:
            ensure_attrs = {k: int(v) if v.isdigit() else v for a in attrs for k, v in [a.split('=')]}
            for k, v in trigger_dynamodb_attr_shortcuts.items():
                if k in ensure_attrs:
                    ensure_attrs[v] = ensure_attrs.pop(k)
            if 'StartingPosition' in ensure_attrs:
                ensure_attrs['StartingPosition'] = ensure_attrs['StartingPosition'].upper()
            if preview:
                stderr(' preview:', table_name)
            else:
                stream_arn = aws.dynamodb.stream_arn(table_name)
                conflict = client('lambda').exceptions.ResourceConflictException
                try:
                    retry(client('lambda').create_event_source_mapping, conflict)(EventSourceArn=stream_arn, FunctionName=name, Enabled=True, **ensure_attrs)
                    stderr('', table_name)
                except conflict as e:
                    *_, kind, uuid = e.args[0].split()
                    resp = client('lambda').get_event_source_mapping(UUID=uuid)
                    for k, v in ensure_attrs.items():
                        if k != 'StartingPosition':
                            assert resp[k] == v, [resp[k], v]
                    stderr('', table_name)
Esempio n. 2
0
def ensure_trigger_cloudwatch(name, arn_lambda, metadata, preview):
    triggers = []
    for trigger in metadata['trigger']:
        if trigger.split()[0] == 'cloudwatch':
            kind, schedule = trigger.split(None, 1)
            triggers.append(schedule)
    if triggers:
        stderr('\nensure triggers cloudwatch:')
        assert len(triggers) == 1, f'only 1 cloudwatch schedule is currently supported: {triggers}'
        for schedule in triggers:
            if preview:
                stderr(' preview:', schedule)
            else:
                arn_rule = client('events').put_rule(Name=name, ScheduleExpression=schedule)['RuleArn']
                ensure_permission(name, 'events.amazonaws.com', arn_rule)
                targets = retry(client('events').list_targets_by_rule)(Rule=name)['Targets']
                assert all(t['Arn'] == arn_lambda for t in targets), f'there are unknown targets in cloudwatch rule: {name}'
                if len(targets) == 0:
                    stderr('', schedule)
                    client('events').put_targets(Rule=name, Targets=[{'Id': '1', 'Arn': arn_lambda}])
                elif len(targets) == 1:
                    assert targets[0]['Arn'] == arn_lambda, f'cloudwatch target mismatch: {arn_lambda} {targets[0]}'
                    stderr('', schedule)
                elif len(targets) > 1:
                    stderr(' removing:', schedule)
                    targets = sorted(targets, key=lambda x: x['Id'])
                    client('events').remove_targets(Rule=name, Ids=[t['Id'] for t in targets[1:]])
                def ensure_only_one_target():
                    targets = client('events').list_targets_by_rule(Rule=name)['Targets']
                    assert len(targets) == 1, f'more than one target found for cloudwatch rule: {name} {schedule} {targets}'
                retry(ensure_only_one_target)()
Esempio n. 3
0
def ls(selectors, state):
    assert state in ['running', 'pending', 'stopped', 'terminated',
                     None], f'bad state: {state}'
    if not selectors:
        instances = list(
            retry(resource('ec2').instances.filter)(Filters=[{
                'Name': 'instance-state-name',
                'Values': [state]
            }] if state else []))
    else:
        kind = 'tags'
        kind = 'dns-name' if selectors[0].endswith('.amazonaws.com') else kind
        kind = 'vpc-id' if selectors[0].startswith('vpc-') else kind
        kind = 'subnet-id' if selectors[0].startswith('subnet-') else kind
        kind = 'instance.group-id' if selectors[0].startswith('sg-') else kind
        kind = 'private-dns-name' if selectors[0].endswith(
            '.ec2.internal') else kind
        kind = 'ip-address' if all(x.isdigit() or x == '.'
                                   for x in selectors[0]) else kind
        kind = 'private-ip-address' if all(
            x.isdigit() or x == '.'
            for x in selectors[0]) and selectors[0].split(
                '.')[0] in ['10', '172'] else kind
        kind = 'instance-id' if selectors[0].startswith('i-') else kind
        if kind == 'tags' and '=' not in selectors[0]:
            # TODO allow passing multiple names, which are summed, and then tags, which are intersected
            selectors = f'Name={selectors[0]}', *selectors[
                1:]  # auto add Name= to the first tag
        instances = []
        for chunk in util.iter.chunk(selectors, 195):  # 200 boto api limit
            filters = [{
                'Name': 'instance-state-name',
                'Values': [state]
            }] if state else []
            try:
                if kind == 'tags':
                    filters += [{
                        'Name': f'tag:{k}',
                        'Values': [v]
                    } for t in chunk for k, v in [t.split('=')]]
                else:
                    filters += [{'Name': kind, 'Values': chunk}]
            except:
                pprint.pprint(selectors)
                pprint.pprint(filters)
                raise
            instances += list(
                retry(resource('ec2').instances.filter)(Filters=filters))
    instances = sorted(instances, key=lambda i: i.instance_id)
    instances = sorted(instances, key=lambda i: tags(i).get('name', 'no-name'))
    instances = sorted(instances,
                       key=lambda i: i.meta.data['LaunchTime'],
                       reverse=True)
    return instances
Esempio n. 4
0
def ensure_key_allows_role(arn_key, arn_role, preview):
    if not preview:
        resp = aws.client('kms').get_key_policy(KeyId=arn_key,
                                                PolicyName='default')
        policy = json.loads(resp['Policy'])
        # ensure that every Statement.Principal.AWS is a list, it can be either a
        # string or a list of strings.
        for statement in policy['Statement']:
            if statement.get('Principal', {}).get('AWS'):
                if isinstance(statement['Principal']['AWS'], str):
                    statement['Principal']['AWS'] = [
                        statement['Principal']['AWS']
                    ]
        # remove invalid principals from all statements, these are caused by the
        # deletion of an iam role referenced by this policy, which transforms the
        # principal from something like "arn:..." to "AIEKFJ...".
        for statement in policy['Statement']:
            if statement.get('Principal', {}).get('AWS'):
                for arn in statement['Principal']['AWS'].copy():
                    if not arn.startswith('arn:'):
                        statement['Principal']['AWS'].remove(arn)
        # ensure that the "allow use of key" Statement contains our role's arn
        for statement in policy['Statement']:
            if statement['Sid'] == 'Allow use of the key':
                if arn_role not in statement['Principal']['AWS']:
                    statement['Principal']['AWS'].append(arn_role)
                break
        # if an "allow use of key" Statement didn't exist, create it
        else:
            policy['Statement'].append({
                "Sid":
                "Allow use of the key",
                "Effect":
                "Allow",
                "Principal": {
                    "AWS": [arn_role]
                },
                "Action": [
                    "kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*",
                    "kms:GenerateDataKey*", "kms:DescribeKey"
                ],
                "Resource":
                "*"
            })
        try:
            retry(aws.client('kms').put_key_policy,
                  silent=True)(KeyId=arn_key,
                               Policy=json.dumps(policy),
                               PolicyName='default')
        except aws.client('kms').exceptions.MalformedPolicyDocumentException:
            stderr(f'fatal: failed to put to key: {arn_key}, policy:\n' +
                   json.dumps(policy, indent=2))
            raise
Esempio n. 5
0
def rm_table(name, print_fn=stderr):
    in_use = client('dynamodb').exceptions.ResourceInUseException
    not_found = client('dynamodb').exceptions.ResourceNotFoundException
    try:
        retry(client('dynamodb').delete_table, in_use,
              not_found)(TableName=name)
    except in_use as e:
        assert str(e).endswith(f'Table is being deleted: {name}'), e
    except not_found:
        pass
    else:
        print_fn('dynamodb deleted:', name)
Esempio n. 6
0
def ensure_permission(name, principal, arn):
    not_found = client('lambda').exceptions.ResourceNotFoundException
    try:
        res = json.loads(retry(client('lambda').get_policy, not_found)(FunctionName=name)['Policy'])
    except not_found:
        statements = []
    else:
        statements = [x['Sid'] for x in res['Statement']]
    id = principal.replace('.', '-')
    if id not in statements:
        client('lambda').add_permission(FunctionName=name, StatementId=id, Action='lambda:InvokeFunction', Principal=principal, SourceArn=arn)
Esempio n. 7
0
def ensure_key(name, arn_user, arn_role, preview):
    stderr('\nensure kms key:')
    if preview:
        stderr(' preview: kms:', name)
    else:
        keys = [
            x for x in all_keys()
            if x['AliasArn'].endswith(f':alias/lambda/{name}')
        ]
        if 0 == len(keys):
            arn_root = ':'.join(arn_user.split(':')[:-1]) + ':root'
            policy = """
            {"Version": "2012-10-17",
             "Statement": [{"Sid": "Enable IAM User Permissions",
                            "Effect": "Allow",
                            "Principal": {"AWS": ["%(arn_user)s", "%(arn_root)s"]},
                            "Action": "kms:*",
                            "Resource": "*"},
                           {"Sid": "Allow use of the key",
                            "Effect": "Allow",
                            "Principal": {"AWS": ["%(arn_user)s", "%(arn_role)s", "%(arn_root)s"]},
                            "Action": ["kms:Encrypt", "kms:Decrypt", "kms:ReEncrypt*", "kms:GenerateDataKey*", "kms:DescribeKey"],
                            "Resource": "*"},
                           {"Sid": "Allow attachment of persistent resources",
                            "Effect": "Allow",
                            "Principal": {"AWS": ["%(arn_user)s", "%(arn_role)s", "%(arn_root)s"]},
                            "Action": ["kms:CreateGrant", "kms:ListGrants", "kms:RevokeGrant"],
                            "Resource": "*",
                            "Condition": {"Bool": {"kms:GrantIsForAWSResource": true}}}]}
            """ % {
                'arn_role': arn_role,
                'arn_user': arn_user,
                'arn_root': arn_root
            }
            _key_id = retry(aws.client('kms').create_key, silent=True)(
                Policy=policy, Description=name)['KeyMetadata']['KeyId']
            aws.client('kms').create_alias(AliasName=f'alias/lambda/{name}',
                                           TargetKeyId=_key_id)
            keys = [
                x for x in all_keys()
                if x['AliasArn'].endswith(f':alias/lambda/{name}')
            ]
            assert len(keys) == 1
            stderr('', keys[0]['AliasArn'])
            return key_id(keys[0])
        elif 1 == len(keys):
            stderr('', keys[0]['AliasArn'])
            return key_id(keys[0])
        else:
            stderr('fatal: found more than 1 key for:', name,
                   '\n' + '\n'.join(keys))
            sys.exit(1)
Esempio n. 8
0
def ensure_table(name, *attrs, preview=False, yes=False, print_fn=stderr):
    # grab some exception shortcuts
    table_exists = client('dynamodb').exceptions.ResourceInUseException
    not_found = client('dynamodb').exceptions.ResourceNotFoundException
    client_error = client('dynamodb').exceptions.ClientError
    # start ensure_attrs with columns
    columns = [attr for attr in attrs if '=' not in attr]
    ensure_attrs = dicts.to_dotted({
        'AttributeDefinitions':
        [{
            'AttributeName': attr_name,
            'AttributeType': attr_type.upper()
        } for column in columns
         for attr_name, attr_type, _ in [column.split(':')]],
        'KeySchema': [{
            'AttributeName': attr_name,
            'KeyType': key_type.upper()
        } for column in columns
                      for attr_name, _, key_type in [column.split(':')]]
    })
    # update ensure_attrs with the rest of the passed attributes
    ensure_attrs.update({
        k: int(v) if v.isdigit() else v
        for attr in attrs if '=' in attr for k, v in [attr.split('=')]
    })
    # resolve any attribute shortcuts
    for k, v in table_attr_shortcuts.items():
        if k in ensure_attrs:
            ensure_attrs[v] = ensure_attrs.pop(k)
    # allow lower case for stream view type
    if 'StreamSpecification.StreamViewType' in ensure_attrs:
        ensure_attrs['StreamSpecification.StreamEnabled'] = True
        ensure_attrs['StreamSpecification.StreamViewType'] = ensure_attrs[
            'StreamSpecification.StreamViewType'].upper()
    # check provisioning and set billing type
    read = ensure_attrs.get('ProvisionedThroughput.ReadCapacityUnits')
    write = ensure_attrs.get('ProvisionedThroughput.WriteCapacityUnits')
    assert (not read and not write) or (
        read and write
    ), 'both read and write must be provisioned, or neither for on-demand'
    ensure_attrs['BillingMode'] = 'PROVISIONED' if read else 'PAY_PER_REQUEST'
    # print and prompt
    print_fn()
    print_fn('TableName:', name)
    for k, v in ensure_attrs.items():
        print_fn(f' {k}: {v}')
    print_fn()
    # fetch existing table attrs
    try:
        existing_attrs = dicts.to_dotted(
            client('dynamodb').describe_table(TableName=name)['Table'])
    # create table
    except not_found:
        # create
        if preview:
            print_fn(' preview: created:', name)
        else:
            if not yes:
                print_fn('\nproceed? y/n ')
                assert sh.getch() == 'y'
            retry(client('dynamodb').create_table, table_exists,
                  client_error)(TableName=name,
                                **dicts.from_dotted(ensure_attrs))
            print_fn(' created:', name)
    # check and maybe update existing table
    else:
        if preview:
            print_fn(' preview: exists:', name)
        else:
            print_fn(' exists:', name)
        # join tags into existing attributes
        existing_attrs.update(
            dicts.to_dotted({
                'Tags': [
                    tag for page in retry(
                        client('dynamodb').get_paginator(
                            'list_tags_of_resource').paginate)(
                                ResourceArn=arn(name)) for tag in page['Tags']
                ]
            }))
        # remap existing attributes to the same schema as table attributes
        existing_attrs = {
            k.replace('BillingModeSummary.', ''): v
            for k, v in existing_attrs.items()
        }
        # check every attribute
        needs_update = False
        for k, v in ensure_attrs.items():
            if v != existing_attrs.get(k):
                needs_update = True
                if preview:
                    print_fn(f' preview: {k}: {existing_attrs.get(k)} -> {v}')
                else:
                    print_fn(f' {k}: {existing_attrs.get(k)} -> {v}')
                assert k.split(
                    '.'
                )[0] != 'KeySchema', 'KeySchema cannot be updated on existing tables'
        # collect tags to remove
        tags_to_remove = []
        for tag in dicts.from_dotted(existing_attrs).get('Tags', []):
            if tag['Key'] not in [
                    t['Key']
                    for t in dicts.from_dotted(ensure_attrs).get('Tags', [])
            ]:
                tags_to_remove.append(tag['Key'])
                if preview:
                    print_fn(f' preview: untag: {tag["Key"]}')
                else:
                    print_fn(f' untag: {tag["Key"]}')
        if not preview:
            # prompt if updates
            if (needs_update or tags_to_remove) and not yes:
                print_fn('\nproceed? y/n ')
                assert sh.getch() == 'y'
            # update if needed
            if needs_update:
                # update tags
                ensure_attrs = dicts.from_dotted(ensure_attrs)
                if 'Tags' in ensure_attrs:
                    for tag in ensure_attrs['Tags']:
                        tag['Value'] = str(tag['Value'])
                    client('dynamodb').tag_resource(ResourceArn=arn(name),
                                                    Tags=ensure_attrs['Tags'])
                    del ensure_attrs['Tags']
                # update table. note: KeySchema cannot be updated
                del ensure_attrs['KeySchema']
                client('dynamodb').update_table(TableName=name, **ensure_attrs)
            # untag if needed
            if tags_to_remove:
                # remove unused tags
                if tags_to_remove:
                    client('dynamodb').untag_resource(ResourceArn=arn(name),
                                                      TagKeys=tags_to_remove)
Esempio n. 9
0
def stream_arn(name):
    not_found = client('dynamodb').exceptions.ResourceNotFoundException
    return retry(client('dynamodb').describe_table,
                 not_found)(TableName=name)['Table']['LatestStreamArn']
Esempio n. 10
0
def arn(name):
    not_found = client('lambda').exceptions.ResourceNotFoundException
    return retry(client('lambda').get_function, not_found)(FunctionName=name)['Configuration']['FunctionArn']