def __init__(self, region_name): """collect info for a region """ if not region_name or region_name not in map(lambda x: x.name, regions()): print("\nError: please specify a valid region name with --region ") print(" valid regions: " + ", ".join(map(lambda x: x.name, regions())) + "\n") exit(1) self.region = region_name self.sg_by_id = {} self.sg_by_name = {} self.queue = Queue() self.service_list = [ "ec2", "elb", "rds", "redshift", "elasticache", "eni" ] try: self.sgs = boto.ec2.connect_to_region( region_name).get_all_security_groups() except Exception as e: print( "\nError: please check your credentials and network connectivity\n" ) exit(1) threads = [] threads.append(Thread(target=self.prepare_sg)) for service in self.service_list: threads.append(Thread(target=self.wrap, args=(service, ))) [x.start() for x in threads] [x.join() for x in threads] while not self.queue.empty(): obj = self.queue.get() self.sg_by_id[obj.sgid]["obj"].add(obj)
def __init__(self, region_name): """collect info for a region """ if not region_name or region_name not in map(lambda x: x.name, regions()): print("\nError: please specify a valid region name with --region ") print(" valid regions: " + ", ".join(map(lambda x: x.name, regions()))+ "\n") exit(1) self.region = region_name self.sg_by_id={} self.sg_by_name={} self.queue = Queue() self.service_list = ["ec2", "elb", "rds", "redshift", "elasticache", "eni"] try: self.sgs = boto.ec2.connect_to_region(region_name).get_all_security_groups() except Exception as e: print("\nError: please check your credentials and network connectivity\n") exit(1) threads = [] threads.append(Thread(target=self.prepare_sg)) for service in self.service_list: threads.append(Thread(target=self.wrap, args=(service,))) [x.start() for x in threads] [x.join() for x in threads] while not self.queue.empty(): obj = self.queue.get() self.sg_by_id[obj.sgid]["obj"].add(obj)
def determine_region(reg): reg_list = [region.name for region in ec2.regions()] if reg in reg_list: return reg else: logging.warning("Unable to get region info. Environment requested: {0}. Regions available: {1}. Returning the default region of us-west-1".format(reg, reg_list)) return 'us-west-1' #Returning us-west-1 as a default region
def slurp(self): """ :returns: item_list - list of ElasticSearchService Items :return: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} for account in self.accounts: for region in regions(): try: if region.name in TROUBLE_REGIONS: continue (client, domains) = self.get_all_es_domains_in_region(account, region) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'es', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(domains), ElasticSearchService.i_am_plural)) for domain in domains: if self.check_ignore_list(domain["DomainName"]): continue # Fetch the policy: item = self.build_item(domain["DomainName"], client, region.name, account, exception_map) if item: item_list.append(item) return item_list, exception_map
def get_from_regions(tags, region=None): """ Search for all instances that match the tags specifed. If the regions parameter is None, then every region is searched, otherwise each the region specified is searched. To search more than one region, pass in a list of regions for region. """ def get_from_one_region(conn, tags): instances = [] logging.info('Searching in region %s' % conn.region.name) reservations = conn.get_all_instances(filters=tags) for reservation in reservations: for instance in reservation.instances: instances.append(instance) return instances instances = [] if region is None: regions = ec2.regions() for region in regions: conn = region.connect() instances += get_from_one_region(conn, tags) else: if not isinstance(region, list): region = [region] for r in region: conn = r.connect() instances += get_from_one_region(conn, tags) if len(instances) == 0: raise Exception('No instances found') return instances
def legacy_instance_type(): with open('legacy_instance_type.csv', 'w+') as cw: csvwriter = csv.writer(cw, delimiter=',') data = [ 'Region Name', 'Instance id', 'Legacy Instance', 'Type', 'Remarks' ] csvwriter.writerow(data) regions = ec2.regions() for reg in regions: if not (reg.name == 'us-gov-west-1' or reg.name == 'cn-north-1'): ec2con = ec2.connect_to_region(reg.name) print reg.name reservations = ec2con.get_all_reservations() for res in reservations: a = res b = a.instances for i in range(len(b)): c = b[i] d = b[i] #Getting the type if the intance c = e = c.instance_type c = str(c) c = c[0:2] data = [ str(reg.name), str(d.id), "Yes", str(e), "Change to new Generation Instances" ] #Checking the instance type if c == "t1" or c == "m1" or c == "c1" or c == "hi1" or c == "m2" or c == "cr1" or c == "hs1": csvwriter.writerow(data)
def connect(self, access_key_id, secret_key, region="us-east-1"): ''' TBD ''' try: _status = 100 _region_list = regions(aws_access_key_id = access_key_id, \ aws_secret_access_key = secret_key) _region_info = False for _idx in range(0, len(_region_list)): if _region_list[_idx].name == region: _region_info = _region_list[_idx] _region_hostname = _region_info.endpoint _msg = "Selected region is " + str(_region_info.name) cbdebug(_msg) break if _region_info: self.ec2conn = _region_info.connect(aws_access_key_id = \ access_key_id, \ aws_secret_access_key = \ secret_key) _status = 0 else: _fmsg = "Unknown EC2 region (" + region + ")" except AWSException, obj: _status = int(obj.error_code) _fmsg = str(obj.error_message)
def get_random_sync_host(level): instances = [] if level not in ('staging', 'prod'): return None regions = ec2.regions() for region in regions: try: conn = ec2.connect_to_region(region.name) for reservation in conn.get_all_instances(): for instance in reservation.instances: instances.append(instance) instances = filter(lambda instance: instance.state == "running", instances) instances = filter(lambda instance: instance.tags.get('Role') == "sync", instances) instances = filter(lambda instance: instance.tags.get('Level') == level, instances) instances = filter(lambda instance: instance.tags.get('Debug') == 'false' , instances) except: print "Unable to connect to region {}".format(region.name) raise instance = random.choice(instances) return instance.tags.get('Name')
def test_from_string_with_a_valid_service_name_declaration(self): sd = ServiceDeclaration() sd.from_string('ec2') expected_regions = [r.name for r in ec2.regions()] assert sd.service_name == 'ec2' assert sd.regions == expected_regions
def regions(self): from boto.ec2 import regions (aws_id, aws_key) = self.credentials return dict((x.name, x) for x in regions(aws_access_key_id=aws_id, aws_secret_access_key=aws_key))
def get_instances_in_regions(regions, filters=None): if not silent and verbose > 0: print "Retrieving instances" if not silent and verbose > 1: print "Regions: %s\nFilters: %s" % (regions, filters) instances_in_regions = [] for region in ec2.regions(): if region.name in regions: if not silent and verbose > 1: print "Connecting %s region" % (region.name) conn = ec2.connect_to_region(region.name, aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key) reservations = conn.get_all_instances(filters=filters) i = 0 for r in reservations: for instance in r.instances: instances_in_regions.append(instance) i += 1 if not silent and verbose > 0: print "Found %d instances in %s region" % (i, region.name) if not silent: print "Got %s instances" % (len(instances_in_regions)) return instances_in_regions
def __init__(self): try: self.connections = [region.connect() for region in regions()] except NoAuthHandlerFound: print "ERROR: use environment variables to set credentials:\n" + \ "AWS_ACCESS_KEY_ID\nAWS_SECRET_ACCESS_KEY" sys.exit()
def getDataByRegions(): regionalData = {} for r in regions(): try: conn = boto.connect_ec2( aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], region=r) reservations = conn.get_all_instances() totalByRegion = 0.0 numStopped = 0 numRunning = 0 for reservation in reservations: for instance in reservation.instances: if instance.state == "stopped": numStopped += 1 else: totalByRegion = totalByRegion + getBurnRate(instance) numRunning += 1 regionalData.update({ "%s" % r.name: { "burnrate": totalByRegion, "numberRunningInstances": numRunning, "numberStoppedInstances": numStopped, "numberAllInstances": numRunning + numStopped } }) except boto.exception.EC2ResponseError: pass return regionalData
def __read_settings(self): ''' Reads the settings from the ec2.ini file ''' self.regions = [] self.config_file = os.path.dirname( os.path.realpath(__file__)) + '/ec2.ini' if (not os.path.exists(self.config_file)): configRegions = self.selected_region self.destination_variable = 'public_dns_name' self.vpc_destination_variable = 'ip_address' else: config = ConfigParser.SafeConfigParser() config.read(self.config_file) configRegions = config.get('ec2', 'regions') self.destination_variable = config.get('ec2', 'destination_variable') self.vpc_destination_variable = config.get( 'ec2', 'vpc_destination_variable') # Regions if (configRegions == 'all'): for regionInfo in ec2.regions(): self.regions.append(regionInfo.name) else: self.regions = configRegions.split(",")
def get_random_sync_host(level): instances = [] if level not in ('staging', 'prod'): return None regions = ec2.regions() for region in regions: try: conn = ec2.connect_to_region(region.name) for reservation in conn.get_all_instances(): for instance in reservation.instances: instances.append(instance) instances = filter(lambda instance: instance.state == "running", instances) instances = filter( lambda instance: instance.tags.get('Role') == "sync", instances) instances = filter( lambda instance: instance.tags.get('Level') == level, instances) instances = filter( lambda instance: instance.tags.get('Debug') == 'false', instances) except: print "Unable to connect to region {}".format(region.name) raise instance = random.choice(instances) return instance.tags.get('Name')
def read_settings(self): ''' Reads the settings from the ec2.ini file ''' config = six.moves.configparser.SafeConfigParser() config.read(self.args.inifile) # is eucalyptus? self.eucalyptus_host = None self.eucalyptus = False if config.has_option('ec2', 'eucalyptus'): self.eucalyptus = config.getboolean('ec2', 'eucalyptus') if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') # Regions self.regions = [] configRegions = config.get('ec2', 'regions') configRegions_exclude = config.get('ec2', 'regions_exclude') if (configRegions == 'all'): if self.eucalyptus_host: self.regions.append( boto.connect_euca(host=self.eucalyptus_host).region.name) else: for regionInfo in ec2.regions(): if regionInfo.name not in configRegions_exclude: self.regions.append(regionInfo.name) else: self.regions = configRegions.split(",") # Destination addresses self.destination_variable = config.get('ec2', 'destination_variable') self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') # Route53 self.route53_enabled = config.getboolean('ec2', 'route53') self.route53_excluded_zones = [] if config.has_option('ec2', 'route53_excluded_zones'): self.route53_excluded_zones.extend( config.get('ec2', 'route53_excluded_zones', '').split(',')) # Cache related if 'EC2_CACHE_PATH' in os.environ: cache_path = os.environ['EC2_CACHE_PATH'] elif self.args.cache_path: cache_path = self.args.cache_path else: cache_path = config.get('ec2', 'cache_path') if not os.path.exists(cache_path): os.makedirs(cache_path) if 'AWS_PROFILE' in os.environ: aws_profile = "{}-".format(os.environ.get('AWS_PROFILE')) else: aws_profile = "" self.cache_path_cache = cache_path + f"/{aws_profile}ansible-ec2.cache" self.cache_path_tags = cache_path + f"/{aws_profile}ansible-ec2.tags.cache" self.cache_path_index = cache_path + f"/{aws_profile}ansible-ec2.index" self.cache_max_age = config.getint('ec2', 'cache_max_age')
def get_ec2(args): ec2_ips = defaultdict(dict) # Connect the region for r in regions(): if r.name == args.region: region = r break else: print "Region %s not found." % args.region sys.exit(1) print "Retrieving EC2 records..." conn = boto.connect_ec2(region=region) reservations = conn.get_all_reservations() for reservation in reservations: instances = reservation.instances for instance in instances: if instance.state == 'running': ec2_ips[ instance.id]['private_ip'] = instance.private_ip_address ec2_ips[instance.id]['public_ip'] = instance.ip_address ec2_ips[instance.id]['private_dns'] = instance.private_dns_name ec2_ips[instance.id]['public_dns'] = instance.public_dns_name return ec2_ips
def read_settings(self): ''' Reads the settings from the ec2.ini file ''' config = ConfigParser.SafeConfigParser() config.read(os.path.dirname(os.path.realpath(__file__)) + '/ec2.ini') # is eucalyptus? self.eucalyptus_host = None self.eucalyptus = False if config.has_option('ec2', 'eucalyptus'): self.eucalyptus = config.getboolean('ec2', 'eucalyptus') if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') # Regions self.regions = [] configRegions = config.get('ec2', 'regions') if (configRegions == 'all'): if self.eucalyptus_host: self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name) else: for regionInfo in ec2.regions(): self.regions.append(regionInfo.name) else: self.regions = configRegions.split(",") # Destination addresses self.destination_variable = config.get('ec2', 'destination_variable') self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') # Cache related cache_path = config.get('ec2', 'cache_path') self.cache_path_cache = cache_path + "/ansible-ec2.cache" self.cache_path_index = cache_path + "/ansible-ec2.index" self.cache_max_age = config.getint('ec2', 'cache_max_age')
def read_settings(self): ''' Reads the settings from the ec2.ini file ''' config = ConfigParser.SafeConfigParser() ec2_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ec2.ini') ec2_ini_path = os.environ.get('EC2_INI_PATH', ec2_default_ini_path) config.read(ec2_ini_path) # is eucalyptus? self.eucalyptus_host = None self.eucalyptus = False if config.has_option('ec2', 'eucalyptus'): self.eucalyptus = config.getboolean('ec2', 'eucalyptus') if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') # Regions self.regions = [] configRegions = config.get('ec2', 'regions') configRegions_exclude = config.get('ec2', 'regions_exclude') if (configRegions == 'all'): if self.eucalyptus_host: self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name) else: for regionInfo in ec2.regions(): if regionInfo.name not in configRegions_exclude: self.regions.append(regionInfo.name) else: self.regions = configRegions.split(",") # Destination addresses self.destination_variable = config.get('ec2', 'destination_variable') self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') # Route53 self.route53_enabled = config.getboolean('ec2', 'route53') self.route53_excluded_zones = [] if config.has_option('ec2', 'route53_excluded_zones'): self.route53_excluded_zones.extend( config.get('ec2', 'route53_excluded_zones', '').split(',')) # Return all EC2/RDS instances if config.has_option('ec2', 'all_instances'): self.all_instances = config.getboolean('ec2', 'all_instances') else: self.all_instances = False if config.has_option('ec2', 'all_rds_instances'): self.all_rds_instances = config.getboolean('ec2', 'all_rds_instances') else: self.all_rds_instances = False # Cache related cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) if not os.path.exists(cache_dir): os.makedirs(cache_dir) self.cache_path_cache = cache_dir + "/ansible-ec2.cache" self.cache_path_index = cache_dir + "/ansible-ec2.index" self.cache_max_age = config.getint('ec2', 'cache_max_age')
def get_ec2_instances_groups(): region = ec2.regions() for reg in region: connection = ec2.connect_to_region(reg.name) if reg.name == "us-gov-west-1" or reg.name == "cn-north-1": print Red + "API call is not allowed in this regions:", reg.name continue reservations = connection.get_all_reservations() sgs = connection.get_all_security_groups() if reservations: for reservation in reservations: for instance in reservation.instances: instance = instance.id for sg in sgs: if len(sg.instances()) != 0: for rule in sg.rules: splitted_sidr = str(rule.grants[0]) if splitted_sidr[-1:] != '0': print Green + "++++++ Found EC2 in region:",reg.name + "\n" + Red + "InstanceID: " + Yellow + "%s" % (instance) + \ "\n" + Red + "IP: " + Green, splitted_sidr[:-1-2], White + "=>" + Purple + " [" + rule.from_port + "]", "[" + rule.to_port + "]" + \ "\n" + Red + "Whois Info: writed to './out.csv' file" + White whois_info(splitted_sidr[:-1 - 2]) print "" else: print White + "Looking Instances in region:", reg.name, "..."
def get_sync_hosts_in_zone(zone, level, include_debug=False): # Hack to make local dev VM work. if zone is None: return [{'name': 'localhost', 'ip_address': '127.0.0.1', 'num_procs': 4}] instances = [] regions = ec2.regions() for region in regions: if not zone.startswith(region.name): continue try: conn = ec2.connect_to_region(region.name) if conn is None: continue for r in conn.get_all_instances(): for i in r.instances: if i.placement != zone: continue if i.tags.get('Role') != 'sync': continue if i.tags.get('Level') != level: continue if not include_debug and i.tags.get('Debug') == 'true': continue instances.append(i) except: print "Unable to connect to region {}".format(region.name) raise return [{ 'name': i.tags.get('Name'), 'ip_address': i.private_ip_address, 'num_procs': num_vcpus(i.instance_type) * 2, 'debug': i.tags.get('Debug') == 'true', } for i in instances]
def get_regions(args, env): if args.all_regions: return [ region.name for region in filter(lambda x: '-gov-' not in x.name, regions()) ] return [env.get_region()]
def parse_cli_args(self): ''' Command line argument processing ''' parser = argparse.ArgumentParser( description='Produce an inventory list based on cached EC2 data', prog='Cached Inventory') parser.add_argument('--version', action='version', version='%(prog)s 2.0') #Customer parser.add_argument('--customer', '-c', required = True) #Display Options parser.add_argument('--verbose', '-v', action='count') parser.add_argument('--hide-header', action = 'store_true', default = False) #Query Options parser.add_argument('--type', choices=['instances', 'volumes', 'snapshots'], required=True) parser.add_argument('--source', '-s', choices=['cache', 'provider'], default = 'cache') #Filters parser.add_argument('--instance', '-i', default = None, help = 'Shows volumes for specified instance') parser.add_argument('--volume', default = None, help = 'Shows snapshots for specified volumes') regions = [r.name for r in ec2.regions()] regions.append('all') parser.add_argument('--ec2-region', default = 'all', choices = regions, help = 'Shows assetts for this region') self.args = parser.parse_args()
def connect(self, access_key_id, secret_key, region = "us-east-1") : ''' TBD ''' try : _status = 100 _region_list = regions(aws_access_key_id = access_key_id, \ aws_secret_access_key = secret_key) _region_info = False for _idx in range(0,len(_region_list)) : if _region_list[_idx].name == region : _region_info = _region_list[_idx] _region_hostname = _region_info.endpoint _msg = "Selected region is " + str(_region_info.name) cbdebug(_msg) break if _region_info : self.ec2conn = _region_info.connect(aws_access_key_id = \ access_key_id, \ aws_secret_access_key = \ secret_key) _status = 0 else : _fmsg = "Unknown EC2 region (" + region + ")" except AWSException, obj: _status = int(obj.error_code) _fmsg = str(obj.error_message)
def test_regions_setter_with_wildcard_list_converts_to_regions_list(self): sd = ServiceDeclaration() sd.service_name = 'ec2' sd.regions = ['*'] expected_regions = [r.name for r in ec2.regions()] assert sd.regions == expected_regions
def getDataByRegions(): regionalData = {} for r in regions(): try: conn = boto.connect_ec2( aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], region = r ) reservations = conn.get_all_instances() totalByRegion = 0.0 numStopped = 0 numRunning = 0 for reservation in reservations: for instance in reservation.instances: if instance.state == "stopped": numStopped += 1 else: totalByRegion = totalByRegion + getBurnRate(instance) numRunning += 1 regionalData.update( {"%s" % r.name: {"burnrate":totalByRegion, "numberRunningInstances":numRunning, "numberStoppedInstances":numStopped, "numberAllInstances":numRunning+numStopped}}) except boto.exception.EC2ResponseError: pass return regionalData
def regions(self): from boto.ec2 import regions (aws_id, aws_key) = self.credentials return dict((x.name, x) for x in regions( aws_access_key_id=aws_id, aws_secret_access_key=aws_key ))
def legacy_instance_type(): with open('legacy_instance_type.csv','w+') as cw: csvwriter=csv.writer(cw,delimiter=',') data=[ 'Region Name','Instance id', 'Legacy Instance','Type','Remarks' ] csvwriter.writerow(data) regions=ec2.regions() for reg in regions: if not (reg.name == 'us-gov-west-1' or reg.name == 'cn-north-1'): ec2con=ec2.connect_to_region(reg.name) print reg.name reservations=ec2con.get_all_reservations() for res in reservations: a=res b=a.instances for i in range (len(b)): c=b[i] d=b[i] #Getting the type if the intance c=e=c.instance_type c=str(c) c=c[0:2] data=[str(reg.name),str(d.id),"Yes",str(e),"Change to new Generation Instances"] #Checking the instance type if c == "t1" or c == "m1" or c == "c1" or c == "hi1" or c == "m2" or c == "cr1" or c == "hs1": csvwriter.writerow(data)
def get_ec2(args): ec2_ips = defaultdict(dict) # Connect the region for r in regions(): if r.name == args.region: region = r break else: print "Region %s not found." % args.region sys.exit(1) print "Retrieving EC2 records..." conn = boto.connect_ec2(region=region) reservations = conn.get_all_reservations() for reservation in reservations: instances = reservation.instances for instance in instances: if instance.state == 'running': # or instance.state == 'stopped': ec2_ips[instance.id]['private_ip'] = instance.private_ip_address ec2_ips[instance.id]['public_ip'] = instance.ip_address ec2_ips[instance.id]['private_dns'] = instance.private_dns_name ec2_ips[instance.id]['public_dns'] = instance.public_dns_name return ec2_ips
def read_settings(self): ''' Reads the settings from the ec2.ini file ''' config = ConfigParser.SafeConfigParser() config.read(self.args.inifile) # is eucalyptus? self.eucalyptus_host = None self.eucalyptus = False if config.has_option('ec2', 'eucalyptus'): self.eucalyptus = config.getboolean('ec2', 'eucalyptus') if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') # Regions self.regions = [] configRegions = config.get('ec2', 'regions') configRegions_exclude = config.get('ec2', 'regions_exclude') if (configRegions == 'all'): if self.eucalyptus_host: self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name) else: for regionInfo in ec2.regions(): if regionInfo.name not in configRegions_exclude: self.regions.append(regionInfo.name) else: self.regions = configRegions.split(",") # Destination addresses self.destination_variable = config.get('ec2', 'destination_variable') self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') # Route53 self.route53_enabled = config.getboolean('ec2', 'route53') self.route53_excluded_zones = [] if config.has_option('ec2', 'route53_excluded_zones'): self.route53_excluded_zones.extend( config.get('ec2', 'route53_excluded_zones', '').split(',')) # Cache related if 'EC2_CACHE_PATH' in os.environ: cache_path = os.environ['EC2_CACHE_PATH'] elif self.args.cache_path: cache_path = self.args.cache_path else: cache_path = config.get('ec2', 'cache_path') if not os.path.exists(cache_path): os.makedirs(cache_path) if 'AWS_PROFILE' in os.environ: aws_profile = "{}-".format(os.environ.get('AWS_PROFILE')) else: aws_profile = "" self.cache_path_cache = cache_path + "/{}ansible-ec2.cache".format(aws_profile) self.cache_path_tags = cache_path + "/{}ansible-ec2.tags.cache".format(aws_profile) self.cache_path_index = cache_path + "/{}ansible-ec2.index".format(aws_profile) self.cache_max_age = config.getint('ec2', 'cache_max_age')
def read_settings(self): ''' Reads the settings from the ec2.ini file ''' config = ConfigParser.SafeConfigParser() ec2_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ec2.ini') ec2_ini_path = os.environ.get('EC2_INI_PATH', ec2_default_ini_path) config.read(ec2_ini_path) boto_aws_config_file = config.get('ec2', 'boto_aws_config') # hack to read aws credentials from a local ini file if os.path.isfile(boto_aws_config_file): creds = ConfigParser.SafeConfigParser() creds.read(boto_aws_config_file) os.environ['AWS_ACCESS_KEY_ID'] = creds.get('aws', 'access_key_id') os.environ['AWS_SECRET_ACCESS_KEY'] = creds.get('aws', 'secret_access_key') # is eucalyptus? self.eucalyptus_host = None self.eucalyptus = False if config.has_option('ec2', 'eucalyptus'): self.eucalyptus = config.getboolean('ec2', 'eucalyptus') if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') # Regions self.regions = [] configRegions = config.get('ec2', 'regions') configRegions_exclude = config.get('ec2', 'regions_exclude') if (configRegions == 'all'): if self.eucalyptus_host: self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name) else: for regionInfo in ec2.regions(): if regionInfo.name not in configRegions_exclude: self.regions.append(regionInfo.name) else: self.regions = configRegions.split(",") # Destination addresses self.destination_variable = config.get('ec2', 'destination_variable') self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') # Route53 self.route53_enabled = config.getboolean('ec2', 'route53') self.route53_excluded_zones = [] if config.has_option('ec2', 'route53_excluded_zones'): self.route53_excluded_zones.extend( config.get('ec2', 'route53_excluded_zones', '').split(',')) # Cache related cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) if not os.path.exists(cache_dir): os.makedirs(cache_dir) self.cache_path_cache = cache_dir + "/ansible-ec2.cache" self.cache_path_index = cache_dir + "/ansible-ec2.index" self.cache_max_age = config.getint('ec2', 'cache_max_age')
def run(self): # Get available regions regions = ec2.regions() # Get AWS info for site,opts in accounts.aws.iteritems(): # Loop accounts self.log('[Running for ' + site + ']', 2) tmp = regions for r in tmp: # Loop regions, per account if hasattr(r, 'name'): self.log('[Connecting to region ' + r.name + ']', 3) # Connect to AWS conn = ec2.connection.EC2Connection( opts['key'], opts['secret'], region=r ) if conn: # Get volumes for this accounts region vols = conn.get_all_volumes() if vols: self.log('[Fetching snapshots]', 4) # Get all available snapshots snapshots = conn.get_all_snapshots(None, 'self') # Loop volumes for vol in vols: mostRecentSnapshots = {} # Check existing snapshots for snp in snapshots: if (snp.volume_id == vol.id) and self.isOneOfOurs(snp): stype = self.getBackupTypeFromDesc(snp) # Snapshot has expired - delete if self.doIneedToDeleteThisSnapshot(snp, opts['backup_timings']): self.log('[Deleting snapshot ' + snp.id + ']', 5) conn.delete_snapshot(snp.id) else: self.log('[Did not need to delete snapshot for ' + snp.id + ']', 5) # Keep a note of the most up to date snapshot (check if we need to make a new one) dt = datetime.datetime.strptime(snp.start_time.replace('.000Z', ''), "%Y-%m-%dT%H:%M:%S") if (stype not in mostRecentSnapshots) or (dt > mostRecentSnapshots[stype]): mostRecentSnapshots[stype] = dt # Loop through backup types for t,v in opts['backup_timings'].iteritems(): if int(v['interval']) > 0: bMakeSnapshot = True if t in mostRecentSnapshots: # Snapshots already exist for this volume - check to see if a new snapshot is required if int(v['interval']) is not 1: if ((mostRecentSnapshots[t] + datetime.timedelta(hours=int(v['interval']))) > datetime.datetime.now()): bMakeSnapshot = False if bMakeSnapshot is True: snapshot_name = self.generateSnapshotName(vol,t) conn.create_snapshot(vol.id, snapshot_name) self.log('[Creating snapshot ' + snapshot_name + ']', 4)
def read_settings(self): ''' Reads the settings from the ec2.ini file ''' config = ConfigParser.SafeConfigParser() ec2_default_ini_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ec2.ini') ec2_ini_path = os.environ.get('EC2_INI_PATH', ec2_default_ini_path) config.read(ec2_ini_path) # Handle multiple AWS accounts self.aws_creds = {} for i in config.sections(): if i != 'ec2': auth = {'key': config.get(i, 'key'), 'secret': config.get(i, 'secret') } self.aws_creds[i] = auth # is eucalyptus? self.eucalyptus_host = None self.eucalyptus = False if config.has_option('ec2', 'eucalyptus'): self.eucalyptus = config.getboolean('ec2', 'eucalyptus') if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') # Regions self.regions = [] configRegions = config.get('ec2', 'regions') configRegions_exclude = config.get('ec2', 'regions_exclude') if (configRegions == 'all'): if self.eucalyptus_host: self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name) else: for regionInfo in ec2.regions(): if regionInfo.name not in configRegions_exclude: self.regions.append(regionInfo.name) else: self.regions = configRegions.split(",") # Destination addresses self.destination_variable = config.get('ec2', 'destination_variable') self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') # Route53 self.route53_enabled = config.getboolean('ec2', 'route53') self.route53_excluded_zones = [] if config.has_option('ec2', 'route53_excluded_zones'): self.route53_excluded_zones.extend( config.get('ec2', 'route53_excluded_zones', '').split(',')) # Cache related self.cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) if not os.path.exists(self.cache_dir): os.makedirs(self.cache_dir) self.cache_max_age = config.getint('ec2', 'cache_max_age')
def get_deployable_regions(prefix='us-'): """Return the deployable regions, filtering for USA. """ regions = ec2.regions() rv = set() for region in regions: if region.name.startswith(prefix): rv.add(region) logger.debug("Got deployable regions: %s" % rv) return rv
def get_regions(pattern): """Fetch and return AWS regions, excluding non-relevant ones. Also exclude the ones whose name does not match the given pattern. """ return [ region for region in ec2.regions() if not region.name.startswith('cn-') and not region.name.startswith( 'us-gov-') and re.search(pattern, region.name) is not None ]
def get_regions_to_copy_to(source_region): ''' Gets the list of regions to copy an AMI to ''' regions = [] for region in ec2.regions(): if region.name == source_region: continue regions.append(region.name) return regions
def slurp(self): """ :returns: item_list - list of EC2 instances. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format(EC2.index, account, region.name)) try: ec2 = connect(account, 'ec2', region=region) all_instances = self.wrap_aws_rate_limited_call( ec2.get_only_instances ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'ec2', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(all_instances), EC2.i_am_plural)) for instance in all_instances: # A list of properties, in order of priority, to try and use as the name names = [instance.tags.get('Name'), instance.private_dns_name, instance.id] for name in names: if name: break if self.check_ignore_list(name): continue # Dont try to track pending, rebooting, terminated, etc, instances as they are missing attributes if instance.state not in ['running', 'stopped']: continue groups = [{'id': group.id, 'name': group.name} for group in instance.groups] instance_info = {'tags': dict(instance.tags), 'type': instance.instance_type, 'vpc_id': instance.vpc_id, 'subnet_id': instance.subnet_id, 'security_groups': groups, 'id': instance.id, 'dns_name': instance.private_dns_name} item = EC2Item(region=region.name, account=account, name=name, config=instance_info) item_list.append(item) return item_list, exception_map
def get_regions(pattern): """Fetch and return AWS regions, excluding non-relevant ones. Also exclude the ones whose name does not match the given pattern. """ return [ region for region in ec2.regions() if not region.name.startswith('cn-') and not region.name.startswith('us-gov-') and re.search(pattern, region.name) is not None ]
def _load_regions(self): valid_regions = ec2.regions() if self.config['regions'] == 'all': for r in valid_regions: if r.name not in self.config['regions_exclude']: self.regions[self._get_region_key(r.name)] = r.name else: for r in valid_regions: if r.name in self.config['regions']: self.regions[self._get_region_key(r.name)] = r.name
def _get_ec2_objects_across_all_regions(func_get_object): ec2_objects = [] aws_regions = regions( aws_acess_key_id=config.CREDENTIALS_AWS_ACCESS_KEY_ID, aws_secret_access_key=config.CREDENTIALS_AWS_SECRET_ACCESS_KEY) for region in aws_regions: ec2_objects.extend(func_get_object(region.name)) return ec2_objects
def _Connect(region): """Verify region and return an EC2 connection object.""" ec2_region = None for r in regions(): if r.name == region: ec2_region = r break assert ec2_region is not None, '"%s" not in the list of ec2 regions: %s' % \ (region, ', '.join(kValidRegions)) return boto.connect_ec2(region=ec2_region)
def main(): parser = OptionParser() parser.add_option("-r", "--region", help="Region (default us-east-1)", dest="region", default="us-east-1") parser.add_option("-H", "--headers", help="Set headers (use 'T:tagname' for including tags)", default=None, action="store", dest="headers", metavar="ID,Zone,Groups,Hostname,State,T:Name") parser.add_option("-t", "--tab", help="Tab delimited, skip header - useful in shell scripts", action="store_true", default=False) parser.add_option("-f", "--filter", help="Filter option sent to DescribeInstances API call, format is key1=value1,key2=value2,...", default=None) (options, args) = parser.parse_args() # Connect the region for r in regions(): if r.name == options.region: region = r break else: print("Region %s not found." % options.region) sys.exit(1) ec2 = boto.connect_ec2(region=region) # Read headers if options.headers: headers = tuple(options.headers.split(',')) else: headers = ("ID", 'Zone', "Groups", "Hostname") # Create format string format_string = "" for h in headers: if h.startswith('T:'): format_string += "%%-%ds" % HEADERS['T:']['length'] else: format_string += "%%-%ds" % HEADERS[h]['length'] # Parse filters (if any) if options.filter: filters = dict([entry.split('=') for entry in options.filter.split(',')]) else: filters = {} # List and print if not options.tab: print(format_string % headers) print("-" * len(format_string % headers)) for r in ec2.get_all_reservations(filters=filters): groups = [g.name for g in r.groups] for i in r.instances: i.groups = ','.join(groups) if options.tab: print("\t".join(tuple(get_column(h, i) for h in headers))) else: print(format_string % tuple(get_column(h, i) for h in headers))
def get_region(region_name): """Get region info by name. :type region_name: basestring :rtype: boto.regioninfo.RegionInfo """ for region in ec2.regions(): if region.name == region_name: return region raise ValueError("Region not found: %s" % region_name)
def main(): args = options() # Connect the region for r in regions(): if r.name == args.region: region = r break else: print "Region %s not found." % args.region sys.exit(1) ec2 = boto.connect_ec2(region=region) # Change the default ebs root volume size root = '/dev/sda' dev_sda1 = boto.ec2.blockdevicemapping.EBSBlockDeviceType( delete_on_termination=True) dev_sda1.size = args.disk_size # size in Gigabytes bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping() bdm[root] = dev_sda1 user_data = """#!/bin/bash resize2fs %s sed -i "s/^HOSTNAME.*/HOSTNAME=%s/g" /etc/sysconfig/network hostname %s """ % (root, args.name, args.name) reservation = ec2.run_instances(args.ami, placement=args.availability_zone, key_name=args.key, instance_type=args.instance_type, security_groups=args.security_group, block_device_map=bdm, user_data=user_data, instance_profile_name=args.iam_role, dry_run=False) instance = reservation.instances[0] print 'Waiting for instance to be running' while instance.state != 'running': print '.', sleep(5) instance.update() print 'done!' instance.add_tag("Name", value=args.name) print "Instance ID: %s" % instance.id print "Public IP: %s" % instance.ip_address print "Private IP: %s" % instance.private_ip_address
def get_instance_map(): instance_map = {} filter_dict = {'instance-state-name': 'running'} for region in ec2.regions(): try: connection = ec2.connect_to_region(region.name) reservations = connection.get_all_reservations(filters=filter_dict, max_results=1000) instances = map_instances(reservations) if len(instances) > 0: instance_map[region.name] = map_instances(reservations) except EC2ResponseError: print("Error connecting to region %s, skipping" % region.name) return instance_map
def get_all_reservations(config): """Return all reservations known to this connection. """ reservations = [] region_list = regions(aws_access_key_id=config.keys.api, aws_secret_access_key=config.keys.secret) for region in region_list: _logger.info("Searching %s", region) cnx = region.connect(aws_access_key_id=config.keys.api, aws_secret_access_key=config.keys.secret) for reservation in cnx.get_all_instances(): _logger.info("Found %s %s", reservation, [str(i.id) for i in reservation.instances]) reservations.append(reservation) return reservations
def update_volumes_tags(filters=None): """Clone tags from instances to volumes. :param filters: apply optional filtering for the :func:`django_fabfile.utils.get_all_instances`. """ for region in regions(): reservations = get_region_conn(region.name).get_all_instances( filters=filters) for res in reservations: inst = res.instances[0] for bdm in inst.block_device_mapping.keys(): vol_id = inst.block_device_mapping[bdm].volume_id vol = inst.connection.get_all_volumes([vol_id])[0] add_tags(vol, inst.tags)
def read_settings(self): ''' Reads the settings from the ec2.ini file ''' config = ConfigParser.SafeConfigParser() config.read(self.args.inifile) # is eucalyptus? self.eucalyptus_host = None self.eucalyptus = False if config.has_option('ec2', 'eucalyptus'): self.eucalyptus = config.getboolean('ec2', 'eucalyptus') if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'): self.eucalyptus_host = config.get('ec2', 'eucalyptus_host') # Regions self.regions = [] configRegions = os.getenv('AWS_REGION') if configRegions is None: configRegions = config.get('ec2', 'regions') configRegions_exclude = config.get('ec2', 'regions_exclude') if (configRegions == 'all'): if self.eucalyptus_host: self.regions.append( boto.connect_euca(host=self.eucalyptus_host).region.name) else: for regionInfo in ec2.regions(): if regionInfo.name not in configRegions_exclude: self.regions.append(regionInfo.name) else: self.regions = configRegions.split(",") # Destination addresses self.destination_variable = config.get('ec2', 'destination_variable') self.backup_destination_variable = config.get( 'ec2', 'backup_destination_variable') # Route53 self.route53_enabled = config.getboolean('ec2', 'route53') self.route53_excluded_zones = [] if config.has_option('ec2', 'route53_excluded_zones'): self.route53_excluded_zones.extend( config.get('ec2', 'route53_excluded_zones', '').split(',')) # Cache related cache_path = config.get('ec2', 'cache_path') self.cache_path_cache = cache_path + "/ansible-ec2.cache" self.cache_path_index = cache_path + "/ansible-ec2.index" self.cache_max_age = config.getint('ec2', 'cache_max_age')
def get_regions_to_copy_to(source_region): ''' Gets the list of regions to copy an AMI to ''' regions = [] for region in ec2.regions(): if region.name == source_region: continue # Filter out GovCloud if region.name == 'us-gov-west-1': continue # Filter out China if region.name == 'cn-north-1': continue regions.append(region.name) return regions
def get_region_conn(region_name=None): """Connect to partially spelled `region_name`. Return connection to default boto region if called without arguments. :param region_name: may be spelled partially.""" creds = config.get_creds() if region_name: matched = [reg for reg in regions(**creds) if re.match(region_name, reg.name)] assert len(matched) > 0, 'No region matches {0}'.format(region_name) assert len(matched) == 1, 'Several regions matches {0}'.format( region_name) return matched[0].connect(**creds) else: return connect_ec2(**creds)
def connect(self, access_key_id, secret_key, region = "us-east-1") : ''' TBD ''' try : _status = 100 _region_list = regions(aws_access_key_id = access_key_id, \ aws_secret_access_key = secret_key) _region_info = False for _idx in range(0,len(_region_list)) : if _region_list[_idx].name == region : _region_info = _region_list[_idx] _region_hostname = _region_info.endpoint _msg = "Selected region is " + str(_region_info.name) cbdebug(_msg) break if _region_info : self.ec2conn = _region_info.connect(aws_access_key_id = \ access_key_id, \ aws_secret_access_key = \ secret_key) _status = 0 else : _fmsg = "Unknown " + self.get_description() + " region (" + region + ")" except AWSException as obj: _status = int(obj.error_code) _fmsg = str(obj.error_message) except Exception as msg : _fmsg = str(msg) _status = 23 finally : if _status : _msg = self.get_description() + " connection failure: " + _fmsg cberr(_msg) raise CldOpsException(_msg, _status) else : _msg = self.get_description() + " connection successful." cbdebug(_msg) return _status, _msg, _region_hostname
def parse_cli_args(self): ''' Command line argument processing ''' parser = argparse.ArgumentParser( description='Produce an inventory list based on cached EC2 data', prog='Cached Inventory') parser.add_argument('--version', action='version', version='%(prog)s 2.0') #Customer parser.add_argument('--customer', '-c', required=True) #Display Options parser.add_argument('--verbose', '-v', action='count') parser.add_argument('--hide-header', action='store_true', default=False) #Query Options parser.add_argument('--type', choices=['instances', 'volumes', 'snapshots'], required=True) parser.add_argument('--source', '-s', choices=['cache', 'provider'], default='cache') #Filters parser.add_argument('--instance', '-i', default=None, help='Shows volumes for specified instance') parser.add_argument('--volume', default=None, help='Shows snapshots for specified volumes') regions = [r.name for r in ec2.regions()] regions.append('all') parser.add_argument('--ec2-region', default='all', choices=regions, help='Shows assetts for this region') self.args = parser.parse_args()