def parse_route_table(self, global_params, region, rt): route_table = {} vpc_id = rt['VpcId'] get_name(rt, route_table, 'VpcId') # TODO: change get_name to have src then dst get_keys(rt, route_table, ['Routes', 'Associations', 'PropagatingVgws']) # Save manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) self.vpcs[vpc_id].route_tables[rt['RouteTableId']] = route_table
def parse_flow_log(self, global_params, region, flow_log): """ :param global_params: :param region: :param flow_log: :return: """ get_name(flow_log, flow_log, 'FlowLogId') flow_log_id = flow_log.pop('FlowLogId') self.flow_logs[flow_log_id] = flow_log
def store_target(self, global_params, region, target): target_type = target.pop('scout2_target_type') if 'VpcId' in target: vpc_id = target.pop('VpcId') manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) tmp = getattr(self, 'vpcs')[vpc_id] target_dict = getattr(tmp, target_type) else: target_dict = getattr(self, target_type) target_id = target[resource_id_map[target_type]] get_name(target, target, resource_id_map[target_type]) target_dict[target_id] = target
def parse_subnet(self, global_params, region, subnet): """ :param global_params: :param region: :param subnet: :return: """ vpc_id = subnet['VpcId'] manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig()) subnet_id = subnet['SubnetId'] get_name(subnet, subnet, 'SubnetId') subnet['flow_logs'] = [] # Save manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig()) self.vpcs[vpc_id].subnets[subnet_id] = subnet
def parse_subnet(self, global_params, region, subnet): """ Parse subnet object. :param global_params: :param region: :param subnet: :return: """ vpc_id = subnet['VpcId'] manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) subnet_id = subnet['SubnetId'] get_name(subnet, subnet, 'SubnetId') # set flow logs that cover this subnet subnet['flow_logs'] = get_subnet_flow_logs_list(self, subnet) # Save manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) self.vpcs[vpc_id].subnets[subnet_id] = subnet
def parse_network_acl(self, global_params, region, network_acl): """ :param global_params: :param region: :param network_acl: :return: """ vpc_id = network_acl['VpcId'] network_acl['id'] = network_acl.pop('NetworkAclId') get_name(network_acl, network_acl, 'id') manage_dictionary(network_acl, 'rules', {}) network_acl['rules']['ingress'] = self.__parse_network_acl_entries(network_acl['Entries'], False) network_acl['rules']['egress'] = self.__parse_network_acl_entries(network_acl['Entries'], True) network_acl.pop('Entries') # Save manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig()) self.vpcs[vpc_id].network_acls[network_acl['id']] = network_acl
def parse_network_acl(self, global_params, region, network_acl): """ :param global_params: :param region: :param network_acl: :return: """ vpc_id = network_acl['VpcId'] network_acl['id'] = network_acl.pop('NetworkAclId') get_name(network_acl, network_acl, 'id') manage_dictionary(network_acl, 'rules', {}) network_acl['rules']['ingress'] = self.__parse_network_acl_entries(network_acl['Entries'], False) network_acl['rules']['egress'] = self.__parse_network_acl_entries(network_acl['Entries'], True) network_acl.pop('Entries') # Save manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) self.vpcs[vpc_id].network_acls[network_acl['id']] = network_acl
def parse_volume(self, global_params, region, volume): """ :param global_params: Parameters shared for all regions :param region: Name of the AWS region :param volume: Single EBS volume :return: """ volume['id'] = volume.pop('VolumeId') volume['name'] = get_name(volume, volume, 'id') self.volumes[volume['id']] = volume
def parse_vpc(self, global_params, region_name, vpc): """ :param global_params: :param region_name: :param vpc: :return: """ vpc_id = vpc['VpcId'] # Save manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig()) self.vpcs[vpc_id].name = get_name(vpc, {}, 'VpcId')
def parse_vpc(self, global_params, region_name, vpc): """ :param global_params: :param region_name: :param vpc: :return: """ vpc_id = vpc['VpcId'] # Save manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types)) self.vpcs[vpc_id].name = get_name(vpc, {}, 'VpcId')
def parse_snapshot(self, global_params, region, snapshot): """ :param global_params: Parameters shared for all regions :param region: Name of the AWS region :param snapshot: Single snapshot :return: """ snapshot['id'] = snapshot.pop('SnapshotId') snapshot['name'] = get_name(snapshot, snapshot, 'id') self.snapshots[snapshot['id']] = snapshot # Get snapshot attribute snapshot['createVolumPermission'] = api_clients[region].describe_snapshot_attribute(Attribute = 'createVolumePermission', SnapshotId = snapshot['id'])['CreateVolumePermissions']
def parse_instance(self, global_params, region, reservation): """ Parse a single EC2 instance :param global_params: Parameters shared for all regions :param region: Name of the AWS region :param instance: Cluster """ for i in reservation['Instances']: instance = {} vpc_id = i['VpcId'] if 'VpcId' in i and i['VpcId'] else ec2_classic manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types)) instance['reservation_id'] = reservation['ReservationId'] instance['id'] = i['InstanceId'] get_name(i, instance, 'InstanceId') get_keys(i, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId']) # Network interfaces & security groups manage_dictionary(instance, 'network_interfaces', {}) for eni in i['NetworkInterfaces']: nic = {} get_keys(eni, nic, ['Association', 'Groups', 'PrivateIpAddresses', 'SubnetId', 'Ipv6Addresses']) instance['network_interfaces'][eni['NetworkInterfaceId']] = nic self.vpcs[vpc_id].instances[i['InstanceId']] = instance
def parse_snapshot(self, global_params, region, snapshot): """ :param global_params: Parameters shared for all regions :param region: Name of the AWS region :param snapshot: Single snapshot :return: """ snapshot['id'] = snapshot.pop('SnapshotId') snapshot['name'] = get_name(snapshot, snapshot, 'id') self.snapshots[snapshot['id']] = snapshot # Get snapshot attribute snapshot['createVolumPermission'] = \ api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', SnapshotId=snapshot['id'])[ 'CreateVolumePermissions']
def main(): # Parse arguments parser = OpinelArgumentParser() parser.add_argument('debug') parser.add_argument('profile') parser.add_argument('force') parser.add_argument('dry-run') parser.add_argument('regions') parser.add_argument('partition-name') parser.parser.add_argument('--interactive', dest='interactive', default=False, action='store_true', help='Interactive prompt to manually enter CIDRs.') parser.parser.add_argument('--csv-ip-ranges', dest='csv_ip_ranges', default=[], nargs='+', help='CSV file(s) containing CIDRs information.') parser.parser.add_argument('--skip-first-line', dest='skip_first_line', default=False, action='store_true', help='Skip first line when parsing CSV file.') parser.parser.add_argument('--attributes', dest='attributes', default=[], nargs='+', help='Name of the attributes to enter for each CIDR.') parser.parser.add_argument('--mappings', dest='mappings', default=[], nargs='+', help='Column number matching attributes when headers differ.') args = parser.parse_args() # Configure the debug level configPrintException(args.debug) # Check version of opinel if not check_requirements(os.path.realpath(__file__)): return 42 # Initialize the list of regions to work with regions = build_region_list('ec2', args.regions, args.partition_name) # For each profile/environment... for profile_name in args.profile: # Interactive mode if args.interactive: # Initalize prefixes attributes = args.attributes filename = 'ip-ranges-%s.json' % profile_name if os.path.isfile(filename): printInfo('Loading existing IP ranges from %s' % filename) prefixes = read_ip_ranges(filename) # Initialize attributes from existing values if attributes == []: for prefix in prefixes: for key in prefix: if key not in attributes: attributes.append(key) else: prefixes = [] # IP prefix does not need to be specified as an attribute attributes = [a for a in attributes if a != 'ip_prefix'] # Prompt for new entries while prompt_4_yes_no('Add a new IP prefix to the ip ranges'): ip_prefix = prompt_4_value('Enter the new IP prefix:') obj = {} for a in attributes: obj[a] = prompt_4_value('Enter the \'%s\' value:' % a) prefixes.append(new_prefix(ip_prefix, obj)) # Support loading from CSV file elif len(args.csv_ip_ranges) > 0: # Initalize prefixes prefixes = [] # Load CSV file contents for filename in args.csv_ip_ranges: with open(filename, 'rt') as f: csv_contents = f.readlines() # Initialize mappings attributes = args.attributes mappings = {} if attributes == []: # Follow structure of first line headers = csv_contents.pop(0).strip().split(',') for index, attribute in enumerate(headers): mappings[attribute] = index elif attributes and args.mappings == []: # Follow structure of first line but only map a subset of fields headers = csv_contents.pop(0).strip().split(',') attributes.append('ip_prefix') for attribute in set(attributes): mappings[attribute] = headers.index(attribute) else: # Indices of columns are provided as an argument for index, attribute in enumerate(attributes): mappings[attribute] = int(args.mappings[index]) if args.skip_first_line: csv_contents.pop(0) # For each line... for line in csv_contents: ip_prefix = {} values = line.strip().split(',') if len(values) < len(mappings): continue for attribute in mappings: ip_prefix[attribute] = values[mappings[attribute]] if 'ip_prefix' in mappings and 'mask' in mappings: ip = ip_prefix.pop('ip_prefix') mask = ip_prefix.pop('mask') ip_prefix['ip_prefix'] = '%s/%s' % (ip, mask.replace('/','')) prefixes.append(ip_prefix) # AWS mode else: # Initialize IP addresses printInfo('Fetching public IP information for the \'%s\' environment...' % profile_name) ip_addresses = {} # Search for AWS credentials credentials = read_creds(profile_name) if not credentials['AccessKeyId']: return 42 # For each region... for region in regions: # Connect to EC2 ec2_client = connect_service('ec2', credentials, region) if not ec2_client: continue # Get public IP addresses associated with EC2 instances printInfo('...in %s: EC2 instances' % region) reservations = handle_truncated_response(ec2_client.describe_instances, {}, ['Reservations']) for reservation in reservations['Reservations']: for i in reservation['Instances']: if 'PublicIpAddress' in i: ip_addresses[i['PublicIpAddress']] = new_ip_info(region, i['InstanceId'], False) get_name(i, ip_addresses[i['PublicIpAddress']], 'InstanceId') if 'NetworkInterfaces' in i: for eni in i['NetworkInterfaces']: if 'Association' in eni: ip_addresses[eni['Association']['PublicIp']] = new_ip_info(region, i['InstanceId'], False) # At that point, we don't know whether it's an EIP or not... get_name(i, ip_addresses[eni['Association']['PublicIp']], 'InstanceId') # Get all EIPs (to handle unassigned cases) printInfo('...in %s: Elastic IP addresses' % region) eips = handle_truncated_response(ec2_client.describe_addresses, {}, ['Addresses']) for eip in eips['Addresses']: instance_id = eip['InstanceId'] if 'InstanceId' in eip else None # EC2-Classic non associated EIPs have an empty string for instance ID (instead of lacking the attribute in VPC) if instance_id == '': instance_id = None ip_addresses[eip['PublicIp']] = new_ip_info(region, instance_id, True) ip_addresses[eip['PublicIp']]['name'] = instance_id # Format prefixes = [] for ip in ip_addresses: prefixes.append(new_prefix(ip, ip_addresses[ip])) # Generate an ip-ranges-<profile>.json file save_ip_ranges(profile_name, prefixes, args.force_write, args.debug)