def sync(account_number, region, vpc_id='', security_group_id=''): cur_date = datetime.datetime.utcnow() client, account_id = get_boto3_resource('ec2', region, account_number=account_number) logging.info('Syncing Security Groups %s %s %s', account_id, region, vpc_id) query = [] if vpc_id: query.append({'Name': 'vpc-id', 'Values': [vpc_id]}) if security_group_id: query.append({'Name': 'group-id', 'Values': [security_group_id]}) added = 0 for page in client.get_paginator('describe_security_groups').paginate(Filters=query): logging.info('Got page with item count %s', len(page['SecurityGroups'])) page_items = [] for item in page['SecurityGroups']: if 'Tags' not in item: item['Tags'] = [] info = { 'date_added': cur_date, 'region': region, 'account_id': item['OwnerId'], 'resource_id': item['GroupId'], 'name': item['GroupName'], 'tags': normalize_tags_list(item['Tags']), 'vpc_id': item['VpcId'], 'vpc_name': db.get_item(models.Vpc, resource_id=item['VpcId'])['name'], 'ingress_rules': get_rules(item['IpPermissions'], "source"), 'egress_rules': get_rules(item['IpPermissionsEgress'], "destination"), } add_tags_as_keys(info, item['Tags']) page_items.append(info) added += 1 db.replace_items(models.SecurityGroup, page_items) logging.info('Addition done') del_query = { 'region': region, 'account_id': str(account_id), 'date_added__ne': cur_date, } if vpc_id: del_query['vpc_id'] = vpc_id if security_group_id: del_query['resource_id'] = security_group_id deleted = db.delete_items(models.SecurityGroup, **del_query) logging.info('Delete done') rsp = {'added': added, 'deleted': deleted} logging.info(rsp) return rsp
def account_list(): if request.method == 'GET': items = db.get_items(models.Account, **request.args) # dont send the secret_key [i.pop('secret_key') for i in items['items']] return items elif request.method == 'POST': data = request.json creds = { 'aws_access_key_id': data.get('access_key', ''), 'aws_secret_access_key': data.get('secret_key', ''), 'role_arn': data.get('role_arn', '') } data['regions'] = data['regions'].split(",") _, account_id = get_boto3_resource('sts', creds=creds) models.Account(account_number=account_id, **data).save() return data['name']
def sync(account_number, region, vpc_id='', instance_id=''): cur_date = datetime.datetime.utcnow() client, account_id = get_boto3_resource('ec2', region, account_number=account_number) logging.info('Syncing Instances %s %s %s', account_id, region, vpc_id) query = [] if vpc_id: query.append({'Name': 'vpc-id', 'Values': [vpc_id]}) if instance_id: query.append({'Name': 'instance-id', 'Values': [instance_id]}) iam, _ = get_boto3_resource('iam', region, account_number=account_number) added = 0 for page in client.get_paginator('describe_instances').paginate( Filters=query): logging.info('Got page with reservation count %s', len(page['Reservations'])) page_items = [] for reservation in page['Reservations']: for item in reservation['Instances']: if item['State']['Name'] == "terminated": continue if 'Tags' not in item: item['Tags'] = [] info = { 'date_added': cur_date, 'region': region, 'account_id': reservation['OwnerId'], 'resource_id': item['InstanceId'], 'name': get_name_tag(item['Tags']), 'tags': normalize_tags_list(item['Tags']), 'instance_type': item['InstanceType'], 'launch_time': item['LaunchTime'], 'key_name': item.get('KeyName', ''), 'vpc_id': item['VpcId'], 'vpc_name': db.get_item(models.Vpc, resource_id=item['VpcId']).get('name', ''), 'iam_instance_profile_arn': item.get('IamInstanceProfile', {}).get('Arn', ''), 'iam_instance_profile_name': item.get('IamInstanceProfile', {}).get('Arn', '').split('/')[-1], 'network_interfaces': get_network_interfaces(item['NetworkInterfaces']), 'az': item['Placement']['AvailabilityZone'], 'state': item['State']['Name'], } set_iam_role(info, iam) add_tags_as_keys(info, item['Tags']) page_items.append(info) added += 1 db.replace_items(models.Instance, page_items) logging.info('Addition done') del_query = { 'region': region, 'account_id': str(account_id), 'date_added__ne': cur_date, } if vpc_id: del_query['vpc_id'] = vpc_id if instance_id: del_query['resource_id'] = instance_id deleted = db.delete_items(models.Instance, **del_query) logging.info('Delete done') rsp = {'added': added, 'deleted': deleted} logging.info(rsp) return rsp
def sync(account_number, region, vpc_id='', route_table_id=''): cur_date = datetime.datetime.utcnow() client, account_id = get_boto3_resource('ec2', region, account_number=account_number) logging.info('Syncing Route tables %s %s %s %s', account_id, region, vpc_id, route_table_id) query = [] if vpc_id: query.append({'Name': 'vpc-id', 'Values': [vpc_id]}) if route_table_id: query.append({'Name': 'route-table-id', 'Values': [route_table_id]}) added = 0 for page in client.get_paginator('describe_route_tables').paginate( Filters=query): logging.info('Got page with item count %s', len(page['RouteTables'])) page_items = [] for item in page['RouteTables']: if 'Tags' not in item: item['Tags'] = [] info = { 'date_added': cur_date, 'region': region, 'account_id': item['OwnerId'], 'resource_id': item['RouteTableId'], 'name': get_name_tag(item['Tags'], item['RouteTableId']), 'tags': normalize_tags_list(item['Tags']), 'vpc_id': item['VpcId'], 'vpc_name': db.get_item(models.Vpc, resource_id=item['VpcId']).get('name', ''), 'subnets': [ assoc['SubnetId'] for assoc in item['Associations'] if not assoc['Main'] ], 'routes': get_routes(item['Routes']), 'main': is_main_rtable(item) } add_tags_as_keys(info, item['Tags']) page_items.append(info) added += 1 db.replace_items(models.RouteTable, page_items) logging.info('Addition done') del_query = { 'region': region, 'account_id': str(account_id), 'date_added__ne': cur_date, } if vpc_id: del_query['vpc_id'] = vpc_id if route_table_id: del_query['resource_id'] = route_table_id deleted = db.delete_items(models.RouteTable, **del_query) logging.info('Delete done') rsp = {'added': added, 'deleted': deleted} logging.info(rsp) return rsp
def sync(account_number, region, vpc_id='', subnet_id=''): cur_date = datetime.datetime.utcnow() client, account_id = get_boto3_resource('ec2', region, account_number=account_number) logging.info('Syncing Subnets %s %s %s', account_id, region, vpc_id) query = [] if vpc_id: query.append({'Name': 'vpc-id', 'Values': [vpc_id]}) if subnet_id: query.append({'Name': 'subnet-id', 'Values': [subnet_id]}) added = 0 for page in client.get_paginator('describe_subnets').paginate( Filters=query): logging.info('Got page with item count %s', len(page['Subnets'])) page_items = [] for item in page['Subnets']: if 'Tags' not in item: item['Tags'] = [] info = { 'date_added': cur_date, 'region': region, 'account_id': item['OwnerId'], 'resource_id': item['SubnetId'], 'name': get_name_tag(item['Tags'], item['SubnetId']), 'tags': normalize_tags_list(item['Tags']), 'vpc_id': item['VpcId'], 'vpc_name': db.get_item(models.Vpc, resource_id=item['VpcId']).get('name', ''), 'cidr': item['CidrBlock'], 'az': item['AvailabilityZone'], 'arn': item['SubnetArn'], 'route_table': get_route_table(region, item['VpcId'], item['SubnetId']) } add_tags_as_keys(info, item['Tags']) page_items.append(info) added += 1 db.replace_items(models.Subnet, page_items) logging.info('Addition done') del_query = { 'region': region, 'account_id': str(account_id), 'date_added__ne': cur_date, } if vpc_id: del_query['vpc_id'] = vpc_id if subnet_id: del_query['resource_id'] = subnet_id deleted = db.delete_items(models.Subnet, **del_query) logging.info('Delete done') rsp = {'added': added, 'deleted': deleted} logging.info(rsp) return rsp
def sync(account_number, region, vpc_id='', lb_name=''): cur_date = datetime.datetime.utcnow() client, account_id = get_boto3_resource('elb', region, account_number=account_number) logging.info('Syncing Classic Load Balancers %s %s %s', account_id, region, vpc_id) lb_names = [] if lb_name: lb_names.append(lb_name) added = 0 for page in client.get_paginator('describe_load_balancers').paginate( LoadBalancerNames=lb_names, PaginationConfig={'PageSize': 20}): logging.info('Got page with item count %s', len(page['LoadBalancerDescriptions'])) page_items = [] for item in page['LoadBalancerDescriptions']: if vpc_id and item['VPCId'] != vpc_id: continue info = { 'date_added': cur_date, 'region': region, 'account_id': account_id, 'resource_id': item['LoadBalancerName'], 'vpc_id': item['VPCId'], 'vpc_name': db.get_item(models.Vpc, vpc_id=item['VPCId'])['name'], 'name': item['LoadBalancerName'], 'type': 'classic', 'scheme': item['Scheme'], 'created_time': item['CreatedTime'], 'subnets': [{ 'resource_id': subnet, 'name': db.get_item(models.Subnet, resource_id=subnet).get('name', '') } for subnet in item['Subnets']], 'security_groups': [{ 'resource_id': sg, 'name': db.get_item(models.SecurityGroup, resource_id=sg).get('name', '') } for sg in item['SecurityGroups']], 'dns': item['DNSName'], 'listeners': get_listeners(item) } page_items.append(info) added += 1 get_lb_tags(client, page_items) db.replace_items(models.LoadBalancer, page_items) logging.info('Addition done') del_query = { 'region': region, 'account_id': str(account_id), 'date_added__ne': cur_date, 'type': 'classic' } if vpc_id: del_query['vpc_id'] = vpc_id if lb_name: del_query['resource_id'] = lb_name deleted = db.delete_items(models.LoadBalancer, **del_query) logging.info('Delete done') rsp = {'added': added, 'deleted': deleted} logging.info(rsp) return rsp
def sync(account_number, region, vpc_id='', lb_name=''): cur_date = datetime.datetime.utcnow() client, account_id = get_boto3_resource('elbv2', region, account_number=account_number) logging.info('Syncing Elastic Load balancers %s %s %s', account_id, region, vpc_id) if not vpc_id: target_groups = get_all_target_groups(client) lb_names = [] if lb_name: lb_names.append(lb_name) added = 0 pagination = {'PageSize': 20} if lb_name: pagination = None for page in client.get_paginator('describe_load_balancers').paginate( Names=lb_names, PaginationConfig=pagination): logging.info('Got page with item count %s', len(page['LoadBalancers'])) page_items = [] for item in page['LoadBalancers']: if item['Type'] != 'network': continue if vpc_id and item['VpcId'] != vpc_id: continue if vpc_id: target_groups = { item['LoadBalancerArn']: client.describe_target_groups( LoadBalancerArn=item['LoadBalancerArn']) ['TargetGroups'] } info = { 'date_added': cur_date, 'region': region, 'account_id': account_id, 'resource_id': item['LoadBalancerName'], 'vpc_id': item['VpcId'], 'vpc_name': db.get_item(models.Vpc, vpc_id=item['VpcId']).get('name', ''), 'name': item['LoadBalancerName'], 'type': item['Type'], 'scheme': item.get('Scheme', ''), 'dns': item.get('DNSName', ''), 'arn': item['LoadBalancerArn'], 'created_time': item['CreatedTime'], 'subnets': [{ 'resource_id': az['SubnetId'], 'name': db.get_item(models.Subnet, resource_id=az['SubnetId']).get('name', '') } for az in item['AvailabilityZones']], 'listeners': get_listeners(client, item['LoadBalancerArn'], target_groups.get(item['LoadBalancerArn'], [])), } page_items.append(info) added += 1 get_lb_tags(client, page_items) db.replace_items(models.LoadBalancer, page_items) logging.info('Addition done') del_query = { 'region': region, 'account_id': account_id, 'date_added__ne': cur_date, 'type': 'network' } if vpc_id: del_query['vpc_id'] = vpc_id if lb_name: del_query['resource_id'] = lb_name deleted = db.delete_items(models.LoadBalancer, **del_query) logging.info('Delete done') rsp = {'added': added, 'deleted': deleted} logging.info(rsp) return rsp