def parse_route_table(self, global_params, region, rt):
route_table = {}
vpc_id = rt['VpcId']
get_name(rt, route_table, 'VpcId') # TODO: change get_name to have src then dst
get_keys(rt, route_table, ['Routes', 'Associations', 'PropagatingVgws'])
# Save
manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types))
self.vpcs[vpc_id].route_tables[rt['RouteTableId']] = route_table
def parse_flow_log(self, global_params, region, flow_log):
"""
:param global_params:
:param region:
:param flow_log:
:return:
"""
get_name(flow_log, flow_log, 'FlowLogId')
flow_log_id = flow_log.pop('FlowLogId')
self.flow_logs[flow_log_id] = flow_log
def parse_route_table(self, global_params, region, rt):
route_table = {}
vpc_id = rt['VpcId']
get_name(rt, route_table,
'VpcId') # TODO: change get_name to have src then dst
get_keys(rt, route_table,
['Routes', 'Associations', 'PropagatingVgws'])
# Save
manage_dictionary(self.vpcs, vpc_id,
SingleVPCConfig(self.vpc_resource_types))
self.vpcs[vpc_id].route_tables[rt['RouteTableId']] = route_table
def parse_flow_log(self, global_params, region, flow_log):
"""
:param global_params:
:param region:
:param flow_log:
:return:
"""
get_name(flow_log, flow_log, 'FlowLogId')
flow_log_id = flow_log.pop('FlowLogId')
self.flow_logs[flow_log_id] = flow_log
def store_target(self, global_params, region, target):
target_type = target.pop('scout2_target_type')
if 'VpcId' in target:
vpc_id = target.pop('VpcId')
manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types))
tmp = getattr(self, 'vpcs')[vpc_id]
target_dict = getattr(tmp, target_type)
else:
target_dict = getattr(self, target_type)
target_id = target[resource_id_map[target_type]]
get_name(target, target, resource_id_map[target_type])
target_dict[target_id] = target
def store_target(self, global_params, region, target):
target_type = target.pop('scout2_target_type')
if 'VpcId' in target:
vpc_id = target.pop('VpcId')
manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types))
tmp = getattr(self, 'vpcs')[vpc_id]
target_dict = getattr(tmp, target_type)
else:
target_dict = getattr(self, target_type)
target_id = target[resource_id_map[target_type]]
get_name(target, target, resource_id_map[target_type])
target_dict[target_id] = target
def parse_subnet(self, global_params, region, subnet):
"""
:param global_params:
:param region:
:param subnet:
:return:
"""
vpc_id = subnet['VpcId']
manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig())
subnet_id = subnet['SubnetId']
get_name(subnet, subnet, 'SubnetId')
subnet['flow_logs'] = []
# Save
manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig())
self.vpcs[vpc_id].subnets[subnet_id] = subnet
def parse_subnet(self, global_params, region, subnet):
"""
Parse subnet object.
:param global_params:
:param region:
:param subnet:
:return:
"""
vpc_id = subnet['VpcId']
manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types))
subnet_id = subnet['SubnetId']
get_name(subnet, subnet, 'SubnetId')
# set flow logs that cover this subnet
subnet['flow_logs'] = get_subnet_flow_logs_list(self, subnet)
# Save
manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types))
self.vpcs[vpc_id].subnets[subnet_id] = subnet
def parse_network_acl(self, global_params, region, network_acl):
"""
:param global_params:
:param region:
:param network_acl:
:return:
"""
vpc_id = network_acl['VpcId']
network_acl['id'] = network_acl.pop('NetworkAclId')
get_name(network_acl, network_acl, 'id')
manage_dictionary(network_acl, 'rules', {})
network_acl['rules']['ingress'] = self.__parse_network_acl_entries(network_acl['Entries'], False)
network_acl['rules']['egress'] = self.__parse_network_acl_entries(network_acl['Entries'], True)
network_acl.pop('Entries')
# Save
manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig())
self.vpcs[vpc_id].network_acls[network_acl['id']] = network_acl
def parse_network_acl(self, global_params, region, network_acl):
"""
:param global_params:
:param region:
:param network_acl:
:return:
"""
vpc_id = network_acl['VpcId']
network_acl['id'] = network_acl.pop('NetworkAclId')
get_name(network_acl, network_acl, 'id')
manage_dictionary(network_acl, 'rules', {})
network_acl['rules']['ingress'] = self.__parse_network_acl_entries(network_acl['Entries'], False)
network_acl['rules']['egress'] = self.__parse_network_acl_entries(network_acl['Entries'], True)
network_acl.pop('Entries')
# Save
manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types))
self.vpcs[vpc_id].network_acls[network_acl['id']] = network_acl
def parse_volume(self, global_params, region, volume):
"""
:param global_params: Parameters shared for all regions
:param region: Name of the AWS region
:param volume: Single EBS volume
:return:
"""
volume['id'] = volume.pop('VolumeId')
volume['name'] = get_name(volume, volume, 'id')
self.volumes[volume['id']] = volume
def parse_subnet(self, global_params, region, subnet):
"""
Parse subnet object.
:param global_params:
:param region:
:param subnet:
:return:
"""
vpc_id = subnet['VpcId']
manage_dictionary(self.vpcs, vpc_id,
SingleVPCConfig(self.vpc_resource_types))
subnet_id = subnet['SubnetId']
get_name(subnet, subnet, 'SubnetId')
# set flow logs that cover this subnet
subnet['flow_logs'] = get_subnet_flow_logs_list(self, subnet)
# Save
manage_dictionary(self.vpcs, vpc_id,
SingleVPCConfig(self.vpc_resource_types))
self.vpcs[vpc_id].subnets[subnet_id] = subnet
def parse_volume(self, global_params, region, volume):
"""
:param global_params: Parameters shared for all regions
:param region: Name of the AWS region
:param volume: Single EBS volume
:return:
"""
volume['id'] = volume.pop('VolumeId')
volume['name'] = get_name(volume, volume, 'id')
self.volumes[volume['id']] = volume
def parse_vpc(self, global_params, region_name, vpc):
"""
:param global_params:
:param region_name:
:param vpc:
:return:
"""
vpc_id = vpc['VpcId']
# Save
manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig())
self.vpcs[vpc_id].name = get_name(vpc, {}, 'VpcId')
def parse_vpc(self, global_params, region_name, vpc):
"""
:param global_params:
:param region_name:
:param vpc:
:return:
"""
vpc_id = vpc['VpcId']
# Save
manage_dictionary(self.vpcs, vpc_id, SingleVPCConfig(self.vpc_resource_types))
self.vpcs[vpc_id].name = get_name(vpc, {}, 'VpcId')
def parse_snapshot(self, global_params, region, snapshot):
"""
:param global_params: Parameters shared for all regions
:param region: Name of the AWS region
:param snapshot: Single snapshot
:return:
"""
snapshot['id'] = snapshot.pop('SnapshotId')
snapshot['name'] = get_name(snapshot, snapshot, 'id')
self.snapshots[snapshot['id']] = snapshot
# Get snapshot attribute
snapshot['createVolumPermission'] = api_clients[region].describe_snapshot_attribute(Attribute = 'createVolumePermission', SnapshotId = snapshot['id'])['CreateVolumePermissions']
def parse_instance(self, global_params, region, reservation):
"""
Parse a single EC2 instance
:param global_params: Parameters shared for all regions
:param region: Name of the AWS region
:param instance: Cluster
"""
for i in reservation['Instances']:
instance = {}
vpc_id = i['VpcId'] if 'VpcId' in i and i['VpcId'] else ec2_classic
manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types))
instance['reservation_id'] = reservation['ReservationId']
instance['id'] = i['InstanceId']
get_name(i, instance, 'InstanceId')
get_keys(i, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId'])
# Network interfaces & security groups
manage_dictionary(instance, 'network_interfaces', {})
for eni in i['NetworkInterfaces']:
nic = {}
get_keys(eni, nic, ['Association', 'Groups', 'PrivateIpAddresses', 'SubnetId', 'Ipv6Addresses'])
instance['network_interfaces'][eni['NetworkInterfaceId']] = nic
self.vpcs[vpc_id].instances[i['InstanceId']] = instance
def parse_instance(self, global_params, region, reservation):
"""
Parse a single EC2 instance
:param global_params: Parameters shared for all regions
:param region: Name of the AWS region
:param instance: Cluster
"""
for i in reservation['Instances']:
instance = {}
vpc_id = i['VpcId'] if 'VpcId' in i and i['VpcId'] else ec2_classic
manage_dictionary(self.vpcs, vpc_id, VPCConfig(self.vpc_resource_types))
instance['reservation_id'] = reservation['ReservationId']
instance['id'] = i['InstanceId']
get_name(i, instance, 'InstanceId')
get_keys(i, instance, ['KeyName', 'LaunchTime', 'InstanceType', 'State', 'IamInstanceProfile', 'SubnetId'])
# Network interfaces & security groups
manage_dictionary(instance, 'network_interfaces', {})
for eni in i['NetworkInterfaces']:
nic = {}
get_keys(eni, nic, ['Association', 'Groups', 'PrivateIpAddresses', 'SubnetId', 'Ipv6Addresses'])
instance['network_interfaces'][eni['NetworkInterfaceId']] = nic
self.vpcs[vpc_id].instances[i['InstanceId']] = instance
def parse_snapshot(self, global_params, region, snapshot):
"""
:param global_params: Parameters shared for all regions
:param region: Name of the AWS region
:param snapshot: Single snapshot
:return:
"""
snapshot['id'] = snapshot.pop('SnapshotId')
snapshot['name'] = get_name(snapshot, snapshot, 'id')
self.snapshots[snapshot['id']] = snapshot
# Get snapshot attribute
snapshot['createVolumPermission'] = \
api_clients[region].describe_snapshot_attribute(Attribute='createVolumePermission', SnapshotId=snapshot['id'])[
'CreateVolumePermissions']
def main():
# Parse arguments
parser = OpinelArgumentParser()
parser.add_argument('debug')
parser.add_argument('profile')
parser.add_argument('force')
parser.add_argument('dry-run')
parser.add_argument('regions')
parser.add_argument('partition-name')
parser.parser.add_argument('--interactive',
dest='interactive',
default=False,
action='store_true',
help='Interactive prompt to manually enter CIDRs.')
parser.parser.add_argument('--csv-ip-ranges',
dest='csv_ip_ranges',
default=[],
nargs='+',
help='CSV file(s) containing CIDRs information.')
parser.parser.add_argument('--skip-first-line',
dest='skip_first_line',
default=False,
action='store_true',
help='Skip first line when parsing CSV file.')
parser.parser.add_argument('--attributes',
dest='attributes',
default=[],
nargs='+',
help='Name of the attributes to enter for each CIDR.')
parser.parser.add_argument('--mappings',
dest='mappings',
default=[],
nargs='+',
help='Column number matching attributes when headers differ.')
args = parser.parse_args()
# Configure the debug level
configPrintException(args.debug)
# Check version of opinel
if not check_requirements(os.path.realpath(__file__)):
return 42
# Initialize the list of regions to work with
regions = build_region_list('ec2', args.regions, args.partition_name)
# For each profile/environment...
for profile_name in args.profile:
# Interactive mode
if args.interactive:
# Initalize prefixes
attributes = args.attributes
filename = 'ip-ranges-%s.json' % profile_name
if os.path.isfile(filename):
printInfo('Loading existing IP ranges from %s' % filename)
prefixes = read_ip_ranges(filename)
# Initialize attributes from existing values
if attributes == []:
for prefix in prefixes:
for key in prefix:
if key not in attributes:
attributes.append(key)
else:
prefixes = []
# IP prefix does not need to be specified as an attribute
attributes = [a for a in attributes if a != 'ip_prefix']
# Prompt for new entries
while prompt_4_yes_no('Add a new IP prefix to the ip ranges'):
ip_prefix = prompt_4_value('Enter the new IP prefix:')
obj = {}
for a in attributes:
obj[a] = prompt_4_value('Enter the \'%s\' value:' % a)
prefixes.append(new_prefix(ip_prefix, obj))
# Support loading from CSV file
elif len(args.csv_ip_ranges) > 0:
# Initalize prefixes
prefixes = []
# Load CSV file contents
for filename in args.csv_ip_ranges:
with open(filename, 'rt') as f:
csv_contents = f.readlines()
# Initialize mappings
attributes = args.attributes
mappings = {}
if attributes == []:
# Follow structure of first line
headers = csv_contents.pop(0).strip().split(',')
for index, attribute in enumerate(headers):
mappings[attribute] = index
elif attributes and args.mappings == []:
# Follow structure of first line but only map a subset of fields
headers = csv_contents.pop(0).strip().split(',')
attributes.append('ip_prefix')
for attribute in set(attributes):
mappings[attribute] = headers.index(attribute)
else:
# Indices of columns are provided as an argument
for index, attribute in enumerate(attributes):
mappings[attribute] = int(args.mappings[index])
if args.skip_first_line:
csv_contents.pop(0)
# For each line...
for line in csv_contents:
ip_prefix = {}
values = line.strip().split(',')
if len(values) < len(mappings):
continue
for attribute in mappings:
ip_prefix[attribute] = values[mappings[attribute]]
if 'ip_prefix' in mappings and 'mask' in mappings:
ip = ip_prefix.pop('ip_prefix')
mask = ip_prefix.pop('mask')
ip_prefix['ip_prefix'] = '%s/%s' % (ip, mask.replace('/',''))
prefixes.append(ip_prefix)
# AWS mode
else:
# Initialize IP addresses
printInfo('Fetching public IP information for the \'%s\' environment...' % profile_name)
ip_addresses = {}
# Search for AWS credentials
credentials = read_creds(profile_name)
if not credentials['AccessKeyId']:
return 42
# For each region...
for region in regions:
# Connect to EC2
ec2_client = connect_service('ec2', credentials, region)
if not ec2_client:
continue
# Get public IP addresses associated with EC2 instances
printInfo('...in %s: EC2 instances' % region)
reservations = handle_truncated_response(ec2_client.describe_instances, {}, ['Reservations'])
for reservation in reservations['Reservations']:
for i in reservation['Instances']:
if 'PublicIpAddress' in i:
ip_addresses[i['PublicIpAddress']] = new_ip_info(region, i['InstanceId'], False)
get_name(i, ip_addresses[i['PublicIpAddress']], 'InstanceId')
if 'NetworkInterfaces' in i:
for eni in i['NetworkInterfaces']:
if 'Association' in eni:
ip_addresses[eni['Association']['PublicIp']] = new_ip_info(region, i['InstanceId'], False) # At that point, we don't know whether it's an EIP or not...
get_name(i, ip_addresses[eni['Association']['PublicIp']], 'InstanceId')
# Get all EIPs (to handle unassigned cases)
printInfo('...in %s: Elastic IP addresses' % region)
eips = handle_truncated_response(ec2_client.describe_addresses, {}, ['Addresses'])
for eip in eips['Addresses']:
instance_id = eip['InstanceId'] if 'InstanceId' in eip else None
# EC2-Classic non associated EIPs have an empty string for instance ID (instead of lacking the attribute in VPC)
if instance_id == '':
instance_id = None
ip_addresses[eip['PublicIp']] = new_ip_info(region, instance_id, True)
ip_addresses[eip['PublicIp']]['name'] = instance_id
# Format
prefixes = []
for ip in ip_addresses:
prefixes.append(new_prefix(ip, ip_addresses[ip]))
# Generate an ip-ranges-<profile>.json file
save_ip_ranges(profile_name, prefixes, args.force_write, args.debug)
