def verify(hosts, config, region, ignore_subnet_check=False):
""" Check DNS entries and IP availability for hosts"""
passed = True
conn = get_aws_connection(region)
for host in hosts:
fqdn = "%s.%s" % (host, config["domain"])
log.info("Checking name conflicts for %s", host)
if not name_available(conn, host):
log.error("%s has been already taken", host)
passed = False
continue
log.debug("Getting IP for %s", fqdn)
ip = get_ip(fqdn)
if not ip:
log.error("%s has no DNS entry", fqdn)
passed = False
else:
log.debug("Getting PTR for %s", fqdn)
ptr = get_ptr(ip)
if ptr != fqdn:
log.error("Bad PTR for %s", host)
passed = False
log.debug("Checking %s availablility", ip)
if not ip_available(region, ip):
log.error("IP %s reserved for %s, but not available", ip, host)
passed = False
if not ignore_subnet_check:
vpc = get_vpc(region)
s_id = get_subnet_id(vpc, ip)
if s_id not in config['subnet_ids']:
log.error("IP %s does not belong to assigned subnets", ip)
passed = False
if not passed:
raise RuntimeError("Sanity check failed")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--config", required=True,
type=argparse.FileType('r'),
help="instance configuration to use")
parser.add_argument("-r", "--region", help="region to use",
default="us-east-1")
parser.add_argument("-n", "--number", type=int, required=True,
help="How many IPs you need")
args = parser.parse_args()
try:
config = json.load(args.config)[args.region]
except KeyError:
parser.error("unknown configuration")
vpc = get_vpc(args.region)
interfaces = vpc.get_all_network_interfaces()
used_ips = [i.private_ip_address for i in interfaces]
subnets = vpc.get_all_subnets(subnet_ids=config["subnet_ids"])
blocks = [s.cidr_block for s in subnets]
available_ips = []
for b in blocks:
# skip first 5 IPs (they are sometimes "reserved") and the last one
# (broadcast)
for ip in list(IP(b))[4:-1]:
if str(ip) not in used_ips:
available_ips.append(ip)
sample = random.sample(available_ips, args.number)
for ip in sample:
print ip
def verify(hosts, config, region, ignore_subnet_check=False):
""" Check DNS entries and IP availability for hosts"""
passed = True
conn = get_aws_connection(region)
for host in hosts:
fqdn = "%s.%s" % (host, config["domain"])
log.info("Checking name conflicts for %s", host)
if not name_available(conn, host):
log.error("%s has been already taken", host)
passed = False
continue
log.debug("Getting IP for %s", fqdn)
ip = get_ip(fqdn)
if not ip:
log.error("%s has no DNS entry", fqdn)
passed = False
else:
log.debug("Getting PTR for %s", fqdn)
ptr = get_ptr(ip)
if ptr != fqdn:
log.error("Bad PTR for %s", host)
passed = False
log.debug("Checking %s availablility", ip)
if not ip_available(region, ip):
log.error("IP %s reserved for %s, but not available", ip, host)
passed = False
if not ignore_subnet_check:
vpc = get_vpc(region)
s_id = get_subnet_id(vpc, ip)
if s_id not in config['subnet_ids']:
log.error("IP %s does not belong to assigned subnets", ip)
passed = False
if not passed:
raise RuntimeError("Sanity check failed")
def get_subnets(regions):
"""
Returns all subnet objects from the specified regions
"""
subnets = []
for r in regions:
log.debug('getting subnets for %s', r)
vpc = get_vpc(r)
subnets.extend(vpc.get_all_subnets())
return subnets
def get_subnets(regions):
"""
Returns all subnet objects from the specified regions
"""
subnets = []
for r in regions:
log.debug('getting subnets for %s', r)
vpc = get_vpc(r)
subnets.extend(vpc.get_all_subnets())
return subnets
def tag_it(i):
log.debug("Tagging %s", i)
vpc = get_vpc(i.region.name)
netif = i.interfaces[0]
# network interface needs to be reloaded usin VPC to get the tags
interface = vpc.get_all_network_interfaces(
filters={"network-interface-id": netif.id})[0]
# copy interface tags over
for tag_name, tag_value in interface.tags.iteritems():
log.info("Adding '%s' tag with '%s' value to %s", tag_name, tag_value,
i)
i.add_tag(tag_name, tag_value)
i.add_tag("moz-state", "ready")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-c",
"--config",
required=True,
type=argparse.FileType('r'),
help="instance configuration to use")
parser.add_argument("-r",
"--region",
help="region to use",
default="us-east-1")
parser.add_argument("-n",
"--number",
type=int,
required=True,
help="How many IPs you need")
args = parser.parse_args()
try:
config = json.load(args.config)[args.region]
except KeyError:
parser.error("unknown configuration")
vpc = get_vpc(args.region)
interfaces = vpc.get_all_network_interfaces()
used_ips = [i.private_ip_address for i in interfaces]
subnets = vpc.get_all_subnets(subnet_ids=config["subnet_ids"])
blocks = [s.cidr_block for s in subnets]
available_ips = []
for b in blocks:
# skip first 5 IPs (they are sometimes "reserved") and the last one
# (broadcast)
for ip in list(IP(b))[4:-1]:
if str(ip) not in used_ips:
available_ips.append(ip)
sample = random.sample(available_ips, args.number)
for ip in sample:
print ip
parser.add_argument("-c", "--config", required=True,
type=argparse.FileType('r'),
help="instance configuration to use")
parser.add_argument("-r", "--region", help="region to use",
default="us-east-1")
parser.add_argument("-n", "--number", type=int, required=True,
help="How many IPs you need")
args = parser.parse_args()
try:
config = json.load(args.config)[args.region]
except KeyError:
parser.error("unknown configuration")
conn = get_aws_connection(args.region)
vpc = get_vpc(args.region)
interfaces = vpc.get_all_network_interfaces()
used_ips = [i.private_ip_address for i in interfaces]
subnets = vpc.get_all_subnets(subnet_ids=config["subnet_ids"])
blocks = [s.cidr_block for s in subnets]
available_ips = []
for b in blocks:
# skip first 5 IPs (they are sometimes "reserved") and the last one
# (broadcast)
for ip in list(IP(b))[4:-1]:
if str(ip) not in used_ips:
available_ips.append(ip)
sample = random.sample(available_ips, args.number)
def create_instance(name, config, region, key_name, instance_data,
deploypass, loaned_to, loan_bug):
"""Creates an AMI instance with the given name and config. The config must
specify things like ami id."""
conn = get_aws_connection(region)
vpc = get_vpc(region)
# Make sure we don't request the same things twice
token = str(uuid.uuid4())[:16]
instance_data = instance_data.copy()
instance_data['name'] = name
instance_data['hostname'] = '{name}.{domain}'.format(
name=name, domain=config['domain'])
ami = conn.get_all_images(image_ids=[config["ami"]])[0]
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
# Overwrite root device size for HVM instances, since they cannot
# be resized online
if ami.virtualization_type == "hvm" and \
ami.root_device_name == device:
bd.size = ami.block_device_mapping[ami.root_device_name].size
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
ip_address = get_ip(instance_data['hostname'])
subnet_id = None
if ip_address:
s_id = get_subnet_id(vpc, ip_address)
if s_id in config['subnet_ids']:
if ip_available(conn, ip_address):
subnet_id = s_id
else:
log.warning("%s already assigned" % ip_address)
if not ip_address or not subnet_id:
ip_address = None
subnet_id = choice(config.get('subnet_ids'))
interface = NetworkInterfaceSpecification(
subnet_id=subnet_id, private_ip_address=ip_address,
delete_on_termination=True,
groups=config.get('security_group_ids', []),
associate_public_ip_address=config.get("use_public_ip")
)
interfaces = NetworkInterfaceCollection(interface)
while True:
try:
reservation = conn.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=token,
disable_api_termination=bool(config.get('disable_api_termination')),
network_interfaces=interfaces,
instance_profile_name=config.get("instance_profile_name"),
)
break
except boto.exception.BotoServerError:
log.exception("Cannot start an instance")
time.sleep(10)
instance = reservation.instances[0]
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
wait_for_status(instance, "state", "running", "update")
instance.add_tag('Name', name)
instance.add_tag('FQDN', instance_data['hostname'])
instance.add_tag('created', time.strftime("%Y-%m-%d %H:%M:%S %Z",
time.gmtime()))
instance.add_tag('moz-type', config['type'])
if loaned_to:
instance.add_tag("moz-loaned-to", loaned_to)
if loan_bug:
instance.add_tag("moz-bug", loan_bug)
log.info("assimilating %s", instance)
instance.add_tag('moz-state', 'pending')
while True:
try:
assimilate(instance.private_ip_address, config, instance_data,
deploypass)
break
except:
log.warn("problem assimilating %s (%s), retrying in 10 sec ...",
instance_data['hostname'], instance.id)
time.sleep(10)
instance.add_tag('moz-state', 'ready')