def client(self, aws_config=None): """Represents the VPCConnection Client """ aws_config_property = (self._get_aws_config_property(aws_config) or self._get_aws_config_from_file()) if not aws_config_property: return VPCConnection() elif aws_config_property.get('ec2_region_name'): region_object = \ get_region(aws_config_property['ec2_region_name']) aws_config = aws_config_property.copy() if region_object and 'ec2_region_endpoint' in aws_config_property: region_object.endpoint = \ aws_config_property['ec2_region_endpoint'] aws_config['region'] = region_object else: aws_config = aws_config_property.copy() if 'ec2_region_name' in aws_config: del (aws_config['ec2_region_name']) # for backward compatibility, # delete this key before passing config to Boto if 'ec2_region_endpoint' in aws_config: del (aws_config["ec2_region_endpoint"]) return VPCConnection(**aws_config)
def _connect_to_region(self, **kwargs): if self._isRegionInfo: return VPCConnection(aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, **kwargs) for region in self.all_region(): if region.name == self.region: self.region = region return VPCConnection(aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, region=self.region, **kwargs)
def create_peering_connections(peering_configs): """ create vpc peering configuration from the peering config dictionary""" vpc_conn = VPCConnection() for peering in peering_configs.keys(): vpc_map = peering_configs[peering]['vpc_map'] vpc_metanetwork_map = peering_configs[peering][ 'vpc_metanetwork_map'] vpc_ids = [vpc.vpc.id for vpc in vpc_map.values()] existing_peerings = vpc_conn.get_all_vpc_peering_connections( filters=[('status-code', 'active'), ('accepter-vpc-info.vpc-id', vpc_ids[0]), ('requester-vpc-info.vpc-id', vpc_ids[1])] ) + vpc_conn.get_all_vpc_peering_connections( filters=[('status-code', 'active'), ('accepter-vpc-info.vpc-id', vpc_ids[1]), ('requester-vpc-info.vpc-id', vpc_ids[0])]) # create peering when peering doesn't exist if not existing_peerings: peering_conn = vpc_conn.create_vpc_peering_connection(*vpc_ids) vpc_conn.accept_vpc_peering_connection(peering_conn.id) logging.info("create new peering connection %s for %s", peering_conn.id, peering) else: peering_conn = existing_peerings[0] logging.info("peering connection %s exists for %s", existing_peerings[0].id, peering) DiscoVPC.create_peering_routes(vpc_conn, vpc_map, vpc_metanetwork_map, peering_conn)
def resolve_security_groups(self): self.log.info("Resolving security groups") # If the server is being spun up in a vpc, search only that vpc exists = lambda s: s in [ group.name for group in self.ec2.get_all_security_groups() if self.vpc_id == group.vpc_id ] for index, group in enumerate(self.security_groups): if not exists(group): self.log.info('Security Group {group} does not exist'.format( group=group)) if self.subnet_id is None: self.ec2.create_security_group(group, group) else: vpc_conn = VPCConnection() vpc_conn.create_security_group(group, group, vpc_id=self.vpc_id) self.log.info( 'Created security group {group}'.format(group=group)) else: self.log.info('Security Group {group} already exists'.format( group=group))
def run(self, terms, variables=None, **kwargs): ret = [] # Below i will implement valid_lookups = { 'id' : None 'dhcp_options_id' : None 'state' : None 'cidr_block' : None 'is_default' : None 'instance_tenancy' : None 'classic_link_enabled' : None } conn = VPCConnection() for term in terms: params = term.split(' ') key_to_lookup = params[0] try: assert( key_to_lookup in valid_lookups ) except (AssertionError) as e: raise AnsibleError(e) vpc_filter = {} for param in params[1:]: tag, value = param.split('=') vpc_filter.update({'tag:'+ tag : value}) vpcs = conn.get_all_vpcs(None, vpc_filter) if len(vpcs) > 1: ret = [ x.get( key_to_lookup ) for x in vpcs ] return ret
def delete_vpc(region, name): conn = VPCConnection(region=region) vpc = get_vpc_by_name(conn, name) if not vpc: print "VPC %s does not exist. Nothing to delete" % name else: delete_sgs_for_vpc(conn, vpc, name) delete_route_tables_for_vpc(conn, vpc, name) delete_igs_for_vpc(conn, vpc, name) delete_subnets_for_vpc(conn, vpc, name) print "Deleting VPC %s..." % name success = False for i in range(0, 100): try: conn.delete_vpc(vpc.id) success = True break except EC2ResponseError as ex: if ex.error_code == "DependencyViolation": print "WARN(%s): Sleeping 2 and retrying" % ex.error_code time.sleep(2) continue else: raise if not success: raise Exception("Could not delete vpc")
def aws_connect(creds_fname=None): credentials_ec2 = creds_fname if not credentials_ec2: config = get_cloudsim_config() # log("config: %s" % config) credentials_ec2 = config['boto_path'] ec2_region_name, aws_access_key_id, aws_secret_access_key, region_endpoint\ = read_boto_file(credentials_ec2) if ec2_region_name == 'nova': # TODO: remove hardcoded OpenStack endpoint region = RegionInfo(None, 'cloudsim', region_endpoint) # 172.16.0.201 ec2conn = EC2Connection(aws_access_key_id, aws_secret_access_key, is_secure=False, region=region, port=8773, path='/services/Cloud') vpcconn = VPCConnection(aws_access_key_id, aws_secret_access_key, is_secure=False, region=region, port=8773, path='/services/Cloud') else: region = RegionInfo(None, ec2_region_name, region_endpoint) ec2conn = boto.connect_ec2(aws_access_key_id, aws_secret_access_key, region=region) vpcconn = boto.connect_vpc(aws_access_key_id, aws_secret_access_key, region=region) return ec2conn, vpcconn
def setUp(self): """ Setup method to initialize vpc_connection objectq """ super(TestVPCConnection, self).setUp() self.vpc_connection = VPCConnection( aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key')
def get_subnet2(name, region): vpc = VPCConnection(aws_access_key_id=access_key, aws_secret_access_key=secret_key) subnet = vpc.get_all_subnets(filters={'tag-value': name}) subnetid = str(subnet) subnetid = (subnetid.split(":"))[1] subnetid = subnetid.replace("]", '') return subnetid
def _configure_environment(self): """Create a new disco style environment VPC""" vpc_cidr = self.get_config("vpc_cidr") # Create VPC vpc_conn = VPCConnection() self.vpc = vpc_conn.create_vpc(self.get_config("vpc_cidr")) keep_trying(300, self.vpc.add_tag, "Name", self.environment_name) keep_trying(300, self.vpc.add_tag, "type", self.environment_type) logging.debug("vpc: %s", self.vpc) dhcp_options = self._configure_dhcp() self.vpc.connection.associate_dhcp_options(dhcp_options.id, self.vpc.id) # Enable DNS vpc_conn.modify_vpc_attribute(self.vpc.id, enable_dns_support=True) vpc_conn.modify_vpc_attribute(self.vpc.id, enable_dns_hostnames=True) # Create metanetworks (subnets, route_tables and security groups) for network in self.networks.itervalues(): network.create() # Configure security group rules for network in self.networks.values(): self._add_sg_rules(network) # Set up security group rules self._open_customer_ports() # Allow ICMP (ping, traceroute & etc) and DNS traffic for all subnets for network in self.networks.itervalues(): self.vpc.connection.authorize_security_group( group_id=network.security_group.id, ip_protocol="icmp", from_port=-1, to_port=-1, cidr_ip=vpc_cidr) self.vpc.connection.authorize_security_group( group_id=network.security_group.id, ip_protocol="udp", from_port=53, to_port=53, cidr_ip=vpc_cidr) # Setup internet gateway internet_gateway = self.vpc.connection.create_internet_gateway() self.vpc.connection.attach_internet_gateway(internet_gateway.id, self.vpc.id) logging.debug("internet_gateway: %s", internet_gateway) self._add_igw_routes(internet_gateway) self._attach_vgw() self.configure_notifications() DiscoVPC.create_peering_connections( DiscoVPC.parse_peerings_config(self.vpc.id)) self.rds.update_all_clusters_in_vpc()
def get_vpc(connection, aws_access_key_id, aws_secret_access_key): global _vpcs if _vpcs.get(connection.region.name): return _vpcs[connection.region.name] _vpcs[connection.region.name] = VPCConnection( region=connection.region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) return _vpcs[connection.region.name]
def vpc_connect(region_name=config['region']): region = boto.ec2.get_region(region_name=region_name) print region vpcconn = VPCConnection(aws_access_key_id=KEY, aws_secret_access_key=SECRET, region=region) print vpcconn return vpcconn
def provision_pinger(args): config = args.config process_config(config) aws_config = config["aws_config"] iam_config = config["iam_config"] s3_config = config["s3_config"] vpc_config = config["vpc_config"] as_config = config["autoscale_config"] elb_config = config["elb_config"] misc_config = config["misc"] load_config_from_s3(s3_config) print "Provisioning Pinger %s" % vpc_config["name"] # create connection conn = VPCConnection(region=aws_config["region"]) # create vpc try: create_iam_users_and_policies(aws_config["region_name"], vpc_config["name"], iam_config) update_pinger_cfg(config) upload_pinger_cfg(vpc_config["name"], s3_config) create_nacho_init_sh(config) vpc = create_vpc(conn, vpc_config["name"], vpc_config["vpc_cidr_block"], vpc_config["instance_tenancy"]) subnet = create_subnet(conn, vpc, vpc_config["name"] + "-SN", vpc_config["subnet_cidr_block"], vpc_config["availability_zone"]) ig = create_ig(conn, vpc, vpc_config["name"] + "-IG") rt = update_route_table(conn, vpc, ig, vpc_config["name"] + "-RT") elb_sg_config = config["elb_config"]["sg_config"] elb_sg = create_sg( conn, vpc, vpc_config["name"] + elb_sg_config["name"] + "-SG", elb_sg_config["description"] + " for " + vpc_config["name"]) add_rules_to_sg(conn, elb_sg, elb_sg_config["ingress-rules"], True, misc_config["jumphost"]) add_rules_to_sg(conn, elb_sg, elb_sg_config["egress-rules"], False, misc_config["jumphost"]) elb = create_elb(aws_config["region_name"], vpc, subnet, elb_sg, vpc_config["name"] + "-ELB", elb_config, s3_config["first_cert_pem"]) ins_sg_config = config["autoscale_config"]["sg_config"] ins_sg = create_sg( conn, vpc, vpc_config["name"] + ins_sg_config["name"] + "-SG", ins_sg_config["description"] + " for " + vpc_config["name"]) add_rules_to_sg(conn, ins_sg, ins_sg_config["ingress-rules"], True, misc_config["jumphost"]) add_rules_to_sg(conn, ins_sg, ins_sg_config["egress-rules"], False, misc_config["jumphost"]) ascaler = create_autoscaler(aws_config["region_name"], vpc, elb, subnet, ins_sg, vpc_config["name"] + "-AS", aws_config, as_config) except (BotoServerError, S3ResponseError, EC2ResponseError) as e: print "Error :%s(%s):%s" % (e.error_code, e.status, e.message) print traceback.format_exc() cleanup(config)
def init_connection(conf): # Load general configuration file cnf = ConfigParser() try: cnf.read(conf) region_os = EC2RegionInfo(endpoint=cnf.get('outscale', 'endpoint')) ows = VPCConnection(cnf.get('outscale', 'access_key'), cnf.get('outscale', 'secret_key'), region=region_os) return ows except Exception as e: print 'error in the configuration file: ', e return None
def delete_peerings(vpc_id=None): """Delete peerings. If vpc_id is specified, delete all peerings of the VPCs only""" vpc_conn = VPCConnection() for peering in DiscoVPC.list_peerings(vpc_id): try: logging.info('deleting peering connection %s', peering.id) vpc_conn.delete_vpc_peering_connection(peering.id) except EC2ResponseError: raise RuntimeError( 'Failed to delete VPC Peering connection {}'.format( peering.id))
def setUpClass(self): time_str = str(int(time.time())) self.route53 = Route53Connection() self.base_domain = 'boto-private-zone-test-%s.com' % time_str self.vpc = VPCConnection() self.test_vpc = self.vpc.create_vpc(cidr_block='10.11.0.0/16') # tag the vpc to make it easily identifiable if things go spang self.test_vpc.add_tag("Name", self.base_domain) self.zone = self.route53.get_zone(self.base_domain) if self.zone is not None: self.zone.delete()
def get_subnet_vpc_id(self, subnet_id): vpc_conn = VPCConnection() subnets = vpc_conn.get_all_subnets(filters={'subnet_id': subnet_id}) if len(subnets) == 1: vpc_id = subnets[0].vpc_id return vpc_id elif len(subnets) == 0: raise NoSubnetReturned( "No subnets returned for: {}".format(subnet_id)) else: raise Exception("More than 1 subnet returned")
def find_vpc_id_by_name(vpc_name): """Find VPC by name""" vpc_conn = VPCConnection() vpc_ids = vpc_conn.get_all_vpcs(filters={'tag:Name': vpc_name}) if len(vpc_ids) == 1: return vpc_ids[0].id elif len(vpc_ids) == 0: raise VPCNameNotFound("No VPC is named as {}".format(vpc_name)) else: raise MultipleVPCsForVPCNameError( "More than 1 VPC is named as {}".format(vpc_name))
def test_db_subnet_group(self): vpc_api = VPCConnection() rds_api = RDSConnection() vpc = vpc_api.create_vpc('10.0.0.0/16') az_list = vpc_api.get_all_zones(filters={'state': 'available'}) subnet = list() n = 0 for az in az_list: try: subnet.append( vpc_api.create_subnet(vpc.id, '10.0.' + str(n) + '.0/24', availability_zone=az.name)) n = n + 1 except: pass grp_name = 'db_subnet_group' + str(int(time.time())) subnet_group = rds_api.create_db_subnet_group( grp_name, grp_name, [subnet[0].id, subnet[1].id]) if not _is_ok(subnet_group, vpc.id, grp_name, [subnet[0].id, subnet[1].id]): raise Exception("create_db_subnet_group returned bad values") rds_api.modify_db_subnet_group(grp_name, description='new description') subnet_grps = rds_api.get_all_db_subnet_groups(name=grp_name) if not _is_ok(subnet_grps[0], vpc.id, 'new description', [subnet[0].id, subnet[1].id]): raise Exception( "modifying the subnet group desciption returned bad values") rds_api.modify_db_subnet_group(grp_name, subnet_ids=[subnet[1].id, subnet[2].id]) subnet_grps = rds_api.get_all_db_subnet_groups(name=grp_name) if not _is_ok(subnet_grps[0], vpc.id, 'new description', [subnet[1].id, subnet[2].id]): raise Exception( "modifying the subnet group subnets returned bad values") rds_api.delete_db_subnet_group(subnet_group.name) try: rds_api.get_all_db_subnet_groups(name=grp_name) raise Exception(subnet_group.name + " still accessible after delete_db_subnet_group") except: pass while n > 0: n = n - 1 vpc_api.delete_subnet(subnet[n].id) vpc_api.delete_vpc(vpc.id)
def connect_vpc(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): """ :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.vpc.VPCConnection` :return: A connection to VPC """ from boto.vpc import VPCConnection return VPCConnection(aws_access_key_id, aws_secret_access_key, **kwargs)
def __init__(self, name, vpc, network_cidr=None, boto3_connection=None): self.vpc = vpc self.name = name if network_cidr: self._network_cidr = IPNetwork(network_cidr) else: self._network_cidr = None self._centralized_route_table_loaded = False self._centralized_route_table = None # lazily initialized self._security_group = None # lazily initialized self._connection = VPCConnection() self._disco_subnets = None # lazily initialized self._boto3_connection = boto3_connection # Lazily initialized if parameter is None
def remove_vpc(self): c = VPCConnection() for subnet in c.get_all_subnets(): if subnet.tags.get('organization') == self.organization: c.delete_subnet(subnet.id) for vpc in c.get_all_vpcs(): if vpc.tags.get('organization') == self.organization: vpc.delete() db_vpc = organization_vpc.OrganizationVPC() org = organizations.Organizations() org.load(name=self.organization_name) db_vpc.load(organization_id=org.get('id')) db_vpc.delete()
def create_nacl(vpc_id, region, network_aclname): vpc = VPCConnection(aws_access_key_id=access_key, aws_secret_access_key=secret_key) network = vpc.create_network_acl(vpc_id) time.sleep(1) ec2_conn = boto.ec2.connect_to_region(region, aws_access_key_id=access_key, aws_secret_access_key=secret_key) ec2_conn.create_tags(network.id, { "Name": network_aclname, "Project": 'LiveLiveProd' }) return network.id
def launch_tag(tag_name, version=keys.v2, reservation_delay=5, pending_delay=5, completion_delay=180): print 'launch_tag:', tag_name conn = boto.ec2.connect_to_region("us-east-1") worker_id = [ sg.id for sg in conn.get_all_security_groups() if sg.name == 'worker' ][0] print 'security group:', worker_id image_id = conn.get_all_images(owners=['self'], filters={'name': version})[0].id print 'ami:', image_id subnet_id = VPCConnection().get_all_subnets()[0].id print 'subnet:', subnet_id interface = boto.ec2.networkinterface.NetworkInterfaceSpecification( subnet_id=subnet_id, groups=[worker_id], associate_public_ip_address=True) interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection( interface) _it = get_size(tag_name) print 'instance type:', _it reservation = conn.run_instances(image_id=image_id, instance_type=_it, network_interfaces=interfaces, key_name='lucid') print 'reservation:', reservation time.sleep(reservation_delay) instance = reservation.instances[0] print 'instance:', instance instance.add_tag("Name", tag_name) instance.update() print 'update tag Name:', tag_name while instance.state == "pending": print 'instance state:', instance, instance.state time.sleep(pending_delay) instance.update() print 'launch done', instance time.sleep(completion_delay)
def get_subnet_availability_zone(self, subnet_id): self.log.info( "getting zone for subnet {subnet_id}".format(subnet_id=subnet_id)) vpc_conn = VPCConnection() filters = {'subnet-id': subnet_id} subnets = vpc_conn.get_all_subnets(filters=filters) if len(subnets) == 1: availability_zone = subnets[0].availability_zone log_message = 'Subnet {subnet_id} is in ' \ 'availability zone {availability_zone}' self.log.info(log_message.format( subnet_id=subnet_id, availability_zone=availability_zone)) return availability_zone
def create_subnet(vpc_id, cidr_block, availability_zone, subnet_name, region): vpc = VPCConnection(aws_access_key_id=access_key, aws_secret_access_key=secret_key) datacenters = vpc.create_subnet(vpc_id=vpc_id, cidr_block=cidr_block, availability_zone=availability_zone) ec2_conn = boto.ec2.connect_to_region(region, aws_access_key_id=access_key, aws_secret_access_key=secret_key) time.sleep(1) ec2_conn.create_tags(datacenters.id, { "Name": subnet_name, "Project": 'LiveLiveProd' }) return datacenters.id
def fetch_environment(cls, vpc_id=None, environment_name=None): """ Returns an instance of this class for the specified VPC, or None if it does not exist """ vpc_conn = VPCConnection() if vpc_id: vpc = vpc_conn.get_all_vpcs(vpc_ids=[vpc_id]) elif environment_name: vpc = vpc_conn.get_all_vpcs(filters={"tag:Name": environment_name}) else: raise VPCEnvironmentError("Expect vpc_id or environment_name") if vpc: vpc = vpc[0] return cls(vpc.tags["Name"], vpc.tags["type"], vpc) else: return None
def parse_peerings_config(vpc_id=None): """ Parses configuration from disco_vpc.ini's peerings sections. If vpc_id is specified, only configuration relevant to vpc_id is included. """ logging.debug("Parsing peerings configuration specified in %s", CONFIG_FILE) vpc_conn = VPCConnection() config = read_config(CONFIG_FILE) if 'peerings' not in config.sections(): logging.info("No VPC peering configuration defined.") return {} peerings = [ peering[1] for peering in config.items('peerings') if peering[0].startswith('connection_') ] for peering in peerings: endpoints = [_.strip() for _ in peering.split(' ')] if len(endpoints) != 2: raise VPCPeeringSyntaxError( "Syntax error in vpc peering connection. " "Expected 2 space-delimited endpoints but found: '{}'". format(peering)) peering_configs = {} for peering in peerings: peering_config = DiscoVPC.parse_peering_connection_line( peering, vpc_conn) vpc_ids_in_peering = [ vpc.vpc.id for vpc in peering_config.get("vpc_map", {}).values() ] if len(vpc_ids_in_peering) < 2: pass # not all vpcs were up, nothing to do elif vpc_id and vpc_id not in vpc_ids_in_peering: logging.debug( "Skipping peering %s because it doesn't include %s", peering, vpc_id) else: peering_configs[peering] = peering_config return peering_configs
def connect_vpc(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): """ :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.vpc.VPCConnection` :return: A connection to VPC """ access_key, secret_key = get_govcloud_creds(aws_access_key_id, aws_secret_access_key) region = RegionInfo(name='govcloud', endpoint='ec2.us-gov-west-1.amazonaws.com') from boto.vpc import VPCConnection return VPCConnection(access_key, secret_key, region=region, **kwargs)
def _get_vpc_details(self): """ Create dictonary with VPC's info and tags """ # Connect to AWS for VPC vpc_conn = VPCConnection() # Grab VPCs vpcs_data = vpc_conn.get_all_vpcs() for vpc in vpcs_data: tags = {} tags['state'] = vpc.state tags['cidr'] = vpc.cidr_block for key, value in vpc.tags.items(): tags[key.lower()] = value.lower() # assing vpcs to class vpcs self.vpcs = {vpc.id: tags}