def setUp(self):
self.private_ip_address1 = PrivateIPAddress(
private_ip_address='10.0.0.10', primary=False)
self.private_ip_address2 = PrivateIPAddress(
private_ip_address='10.0.0.11', primary=False)
self.network_interfaces_spec1 = NetworkInterfaceSpecification(
device_index=1,
subnet_id='subnet_id',
description='description1',
private_ip_address='10.0.0.54',
delete_on_termination=False,
private_ip_addresses=[
self.private_ip_address1, self.private_ip_address2
])
self.private_ip_address3 = PrivateIPAddress(
private_ip_address='10.0.1.10', primary=False)
self.private_ip_address4 = PrivateIPAddress(
private_ip_address='10.0.1.11', primary=False)
self.network_interfaces_spec2 = NetworkInterfaceSpecification(
device_index=2,
subnet_id='subnet_id2',
description='description2',
groups=['group_id1', 'group_id2'],
private_ip_address='10.0.1.54',
delete_on_termination=False,
private_ip_addresses=[
self.private_ip_address3, self.private_ip_address4
],
associate_public_ip_address=True)
def test_associate_elastic_ip(self):
interface = NetworkInterfaceSpecification(
associate_public_ip_address=False,
subnet_id=self.subnet.id,
# Just for testing.
delete_on_termination=True)
interfaces = NetworkInterfaceCollection(interface)
reservation = self.api.run_instances(image_id='ami-a0cd60c9',
instance_type='m1.small',
network_interfaces=interfaces)
instance = reservation.instances[0]
# Register instance to be removed
self.instances.append(instance)
# Add terminate instances helper as cleanup command
self.addCleanup(self.terminate_instances)
# Create an internet gateway so we can attach an eip
igw = self.api.create_internet_gateway()
# Wait on gateway before attaching
time.sleep(5)
# Attach and register clean up tasks
self.api.attach_internet_gateway(igw.id, self.vpc.id)
self.post_terminate_cleanups.append(
(self.api.detach_internet_gateway, (igw.id, self.vpc.id)))
self.post_terminate_cleanups.append(
(self.api.delete_internet_gateway, (igw.id, )))
# Allocate an elastic ip to this vpc
eip = self.api.allocate_address('vpc')
self.post_terminate_cleanups.append((self.delete_elastic_ip, (eip, )))
# Wait on instance and eip then try to associate directly to instance
time.sleep(60)
eip.associate(instance.id)
def test_run_instance_with_nic_preexisting():
conn = boto.connect_vpc('the_key', 'the_secret')
vpc = conn.create_vpc("10.0.0.0/16")
subnet = conn.create_subnet(vpc.id, "10.0.0.0/18")
security_group1 = conn.create_security_group('test security group #1', 'this is a test security group')
security_group2 = conn.create_security_group('test security group #2', 'this is a test security group')
private_ip = "54.0.0.1"
eni = conn.create_network_interface(subnet.id, private_ip, groups=[security_group1.id])
# Boto requires NetworkInterfaceCollection of NetworkInterfaceSpecifications...
# annoying, but generates the desired querystring.
from boto.ec2.networkinterface import NetworkInterfaceSpecification, NetworkInterfaceCollection
interface = NetworkInterfaceSpecification(network_interface_id=eni.id, device_index=0)
interfaces = NetworkInterfaceCollection(interface)
# end Boto objects
reservation = conn.run_instances('ami-1234abcd', network_interfaces=interfaces,
security_group_ids=[security_group2.id])
instance = reservation.instances[0]
instance.subnet_id.should.equal(subnet.id)
all_enis = conn.get_all_network_interfaces()
all_enis.should.have.length_of(1)
instance.interfaces.should.have.length_of(1)
instance_eni = instance.interfaces[0]
instance_eni.id.should.equal(eni.id)
instance_eni.subnet_id.should.equal(subnet.id)
instance_eni.groups.should.have.length_of(2)
set([group.id for group in instance_eni.groups]).should.equal(set([security_group1.id,security_group2.id]))
instance_eni.private_ip_addresses.should.have.length_of(1)
instance_eni.private_ip_addresses[0].private_ip_address.should.equal(private_ip)
def start_master(conn, opts, cluster_name, master_group):
try:
conn.get_all_images(image_ids=[opts.ami])[0]
except boto.exception.EC2ResponseError:
print("Could not find AMI " + opts.ami)
sys.exit(1)
if opts.vpc_id:
interface = NetworkInterfaceSpecification(
subnet_id=opts.subnet_id,
groups=[master_group.id],
associate_public_ip_address=True)
interfaces = NetworkInterfaceCollection(interface)
security_group_ids = None
else:
interfaces = None
security_group_ids = [master_group.id]
master_res = conn.run_instances(image_id=opts.ami,
key_name=opts.key_pair,
instance_type=opts.master_instance_type,
placement=opts.zone,
min_count=1,
max_count=1,
network_interfaces=interfaces,
security_group_ids=security_group_ids)
instance = master_res.instances[0]
time.sleep(1)
conn.create_tags([instance.id],
{"Name": "{c}-master".format(c=cluster_name)})
return instance
def make_instance_interfaces(region, hostname, ignore_subnet_check,
avail_subnets, security_groups, use_public_ip):
vpc = get_vpc(region)
ip_address = get_ip(hostname)
subnet_id = None
if ip_address:
log.info("Using IP %s", ip_address)
s_id = get_subnet_id(vpc, ip_address)
log.info("subnet %s", s_id)
if ignore_subnet_check:
log.info("ignore_subnet_check, using %s", s_id)
subnet_id = s_id
elif s_id in avail_subnets:
if ip_available(region, ip_address):
subnet_id = s_id
else:
log.warning("%s already assigned" % ip_address)
if not ip_address or not subnet_id:
ip_address = None
log.info("Picking random IP")
subnet_id = random.choice(avail_subnets)
interface = NetworkInterfaceSpecification(
subnet_id=subnet_id,
private_ip_address=ip_address,
delete_on_termination=True,
groups=security_groups,
associate_public_ip_address=use_public_ip)
return NetworkInterfaceCollection(interface)
def test_associate_public_ip(self):
# Supplying basically nothing ought to work.
interface = NetworkInterfaceSpecification(
associate_public_ip_address=True,
subnet_id=self.subnet.id,
# Just for testing.
delete_on_termination=True)
interfaces = NetworkInterfaceCollection(interface)
reservation = self.api.run_instances(image_id='ami-a0cd60c9',
instance_type='m1.small',
network_interfaces=interfaces)
instance = reservation.instances[0]
self.instances.append(instance)
self.addCleanup(self.terminate_instances)
# Give it a **LONG** time to start up.
# Because the public IP won't be there right away.
time.sleep(60)
retrieved = self.api.get_all_reservations(instance_ids=[instance.id])
self.assertEqual(len(retrieved), 1)
retrieved_instances = retrieved[0].instances
self.assertEqual(len(retrieved_instances), 1)
retrieved_instance = retrieved_instances[0]
self.assertEqual(len(retrieved_instance.interfaces), 1)
interface = retrieved_instance.interfaces[0]
# There ought to be a public IP there.
# We can't reason about the IP itself, so just make sure it vaguely
# resembles an IP (& isn't empty/``None``)...
self.assertTrue(interface.publicIp.count('.') >= 3)
def create_instance(create_list):
region = create_list["region"]
subnet_id = create_list["subnet_id"]
ami_id = create_list["ami_id"]
key_name = create_list["key_name"]
instance_type = create_list["instance_type"]
sg_id = create_list["sg_id"]
user_data = create_list["user_data"]
usage = create_list["usage"]
#############################
# some user_data oper here
#############################
network_interface = NetworkInterfaceSpecification(subnet_id=subnet_id,groups=[sg_id])
network_interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(network_interface)
conn = boto.ec2.connect_to_region(region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
reservation = conn.run_instances(ami_id,
key_name=key_name,
network_interfaces=network_interfaces,
instance_type=instance_type,
min_count=1,
max_count=1,
user_data=user_data
)
def _request_spot_instances(self, price, availability_zone_group,
instance_type, kwargs):
kwargs.self = None
kwargs.kwargs = None
# m3 INSTANCES ARE NOT ALLOWED PLACEMENT GROUP
if instance_type.startswith("m3."):
kwargs.placement_group = None
kwargs.network_interfaces = NetworkInterfaceCollection(
*(NetworkInterfaceSpecification(**i)
for i in listwrap(kwargs.network_interfaces)
if self.vpc_conn.get_all_subnets(
subnet_ids=i.subnet_id,
filters={"availabilityZone": availability_zone_group})))
if len(kwargs.network_interfaces) == 0:
Log.error(
"No network interface specifications found for {{availability_zone}}!",
availability_zone=kwargs.availability_zone_group)
block_device_map = BlockDeviceMapping()
# GENERIC BLOCK DEVICE MAPPING
for dev, dev_settings in kwargs.block_device_map.items():
block_device_map[dev] = BlockDeviceType(delete_on_termination=True,
**dev_settings)
kwargs.block_device_map = block_device_map
# INCLUDE EPHEMERAL STORAGE IN BlockDeviceMapping
num_ephemeral_volumes = ephemeral_storage[instance_type]["num"]
for i in range(num_ephemeral_volumes):
letter = convert.ascii2char(98 + i) # START AT "b"
kwargs.block_device_map["/dev/sd" + letter] = BlockDeviceType(
ephemeral_name='ephemeral' + text(i),
delete_on_termination=True)
if kwargs.expiration:
kwargs.valid_until = (Date.now() +
Duration(kwargs.expiration)).format(ISO8601)
kwargs.expiration = None
# ATTACH NEW EBS VOLUMES
for i, drive in enumerate(self.settings.utility[instance_type].drives):
letter = convert.ascii2char(98 + i + num_ephemeral_volumes)
device = drive.device = coalesce(drive.device, "/dev/sd" + letter)
d = drive.copy()
d.path = None # path AND device PROPERTY IS NOT ALLOWED IN THE BlockDeviceType
d.device = None
if d.size:
kwargs.block_device_map[device] = BlockDeviceType(
delete_on_termination=True, **d)
output = list(self.ec2_conn.request_spot_instances(**kwargs))
return output
def create_instance():
#
networks = NetworkInterfaceSpecification(subnet_id=subnet_id,
groups=groups,
associate_public_ip_address=True)
network_interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(
networks)
conn = boto.ec2.connect_to_region(ec2_region)
reservation = conn.run_instances(
ami_id,
key_name=key_name,
network_interfaces=network_interfaces,
instance_type=instance_type,
min_count=1,
max_count=1,
block_device_map=block_device_map,
user_data=user_data,
)
instance = reservation.instances[0]
## add tags
instance.add_tags(tags)
## allocate eip
allocate_address = conn.allocate_address(domain='vpc', dry_run=False)
eip = allocate_address.public_ip
allocation_id = allocate_address.allocation_id
## get instance information
print "Instance is launching, Please wait... "
time_init = 0
time_total = 300
time_interval = 5
while time_init < time_total:
status = instance.update()
if status == 'running':
conn.associate_address(instance_id=instance.id,
public_ip=eip,
allocation_id=allocation_id,
network_interface_id=None,
private_ip_address=None,
allow_reassociation=False,
dry_run=False)
print "InstanceID:\t%s" % instance.id
print "Placement:\t%s" % instance.placement
print "InstanceType:\t%s" % instance.instance_type
print "PrivateIp:\t%s" % instance.private_ip_address
print "PublicIp:\t%s" % instance.ip_address
print "ElasticIp:\t%s" % eip
break
else:
time.sleep(time_interval)
time_init += time_interval
def create_interfaces_specification(self, subnets=None, public_ip=False):
"""
Create a network interface specification for an instance -- to be used
with run_instance()
"""
random_subnet = choice(subnets if subnets else self.subnets)
interface = NetworkInterfaceSpecification(
subnet_id=random_subnet.id,
groups=[self.security_group.id],
associate_public_ip_address=public_ip)
interfaces = NetworkInterfaceCollection(interface)
return interfaces
def auto_assign_ip(self):
"""
Calling this method modifies network configuration for the instance so that the instance gets a public
IP address.
"""
interface = NetworkInterfaceSpecification(
subnet_id=self.subnet,
groups=self.security_groups,
associate_public_ip_address=True)
self.interfaces = NetworkInterfaceCollection(interface)
return self
def launch(self, wait=False):
self.security_group_ids = self.get_security_group_ids(
self.security_groups, self.vpc_id)
self.log.info("Using Security group ids: {ids}".format(
ids=self.security_group_ids))
parameters = {
'image_id': self.ami,
'instance_profile_name': self.role,
'key_name': self.keypair,
'instance_type': self.instance_type,
'block_device_map': self.blockdevicemapping,
'user_data': self.user_data,
'ebs_optimized': self.ebs_optimized
}
if self.subnet_id is None:
parameters.update({
'placement': self.availability_zone,
'security_group_ids': self.security_group_ids,
})
else:
interface = NetworkInterfaceSpecification(
subnet_id=self.subnet_id,
groups=self.security_group_ids,
associate_public_ip_address=True)
interfaces = NetworkInterfaceCollection(interface)
parameters.update({'network_interfaces': interfaces})
reservation = self.ec2.run_instances(**parameters)
self.log.info('Successfully launched EC2 instance')
self.instance = reservation.instances[0]
if wait:
self.log.info('Waiting until the instance is running to return')
state = self.instance.state
while not (state == 'running'):
try:
self.instance.update()
state = self.instance.state
except Exception:
pass
self.log.info('The instance is running')
return
def create_instance(connection, instance_name, config, key_name):
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bdm[device] = BlockDeviceType(size=device_info['size'],
delete_on_termination=True)
if 'user_data_file' in config:
log.debug("reading user_data from '%s'" % config['user_data_file'])
user_data = open(config['user_data_file']).read()
# assert that there are no values in need of formatting
user_data = user_data.format()
else:
user_data = None
subnet_id = random.choice(config.get('subnet_ids'))
interface = NetworkInterfaceSpecification(
subnet_id=subnet_id,
delete_on_termination=True,
groups=config.get('security_group_ids', []),
associate_public_ip_address=config.get("use_public_ip"))
interfaces = NetworkInterfaceCollection(interface)
reservation = connection.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=str(uuid.uuid4())[:16],
disable_api_termination=bool(config.get('disable_api_termination')),
user_data=user_data,
instance_profile_name=config.get('instance_profile_name'),
network_interfaces=interfaces,
)
instance = reservation.instances[0]
instance.add_tag('Name', instance_name)
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
wait_for_status(instance, 'state', 'running', 'update')
log.info("instance %s is running; waiting for shutdown", instance)
wait_for_status(instance, 'state', 'stopped', 'update')
log.info("clearing userData")
instance.modify_attribute("userData", None)
return instance
def create_interfaces_specification(self,
subnet_ids=None,
public_ip=False):
"""
Create a network interface specification for an instance -- to be used
with run_instance()
"""
random_subnet_id = choice(subnet_ids if subnet_ids else [
disco_subnet.subnet_dict['SubnetId']
for disco_subnet in self.disco_subnets.values()
])
interface = NetworkInterfaceSpecification(
subnet_id=random_subnet_id,
groups=[self.security_group.id],
associate_public_ip_address=public_ip)
interfaces = NetworkInterfaceCollection(interface)
return interfaces
def test_multi_ip_create(self):
interface = NetworkInterfaceSpecification(
device_index=0,
subnet_id=self.subnet.id,
private_ip_address='10.0.0.21',
description="This is a test interface using boto.",
delete_on_termination=True,
private_ip_addresses=[
PrivateIPAddress(private_ip_address='10.0.0.22',
primary=False),
PrivateIPAddress(private_ip_address='10.0.0.23',
primary=False),
PrivateIPAddress(private_ip_address='10.0.0.24', primary=False)
])
interfaces = NetworkInterfaceCollection(interface)
reservation = self.api.run_instances(image_id='ami-a0cd60c9',
instance_type='m1.small',
network_interfaces=interfaces)
# Give it a few seconds to start up.
time.sleep(10)
instance = reservation.instances[0]
self.addCleanup(self.terminate_instance, instance)
retrieved = self.api.get_all_reservations(instance_ids=[instance.id])
self.assertEqual(len(retrieved), 1)
retrieved_instances = retrieved[0].instances
self.assertEqual(len(retrieved_instances), 1)
retrieved_instance = retrieved_instances[0]
self.assertEqual(len(retrieved_instance.interfaces), 1)
interface = retrieved_instance.interfaces[0]
private_ip_addresses = interface.private_ip_addresses
self.assertEqual(len(private_ip_addresses), 4)
self.assertEqual(private_ip_addresses[0].private_ip_address,
'10.0.0.21')
self.assertEqual(private_ip_addresses[0].primary, True)
self.assertEqual(private_ip_addresses[1].private_ip_address,
'10.0.0.22')
self.assertEqual(private_ip_addresses[2].private_ip_address,
'10.0.0.23')
self.assertEqual(private_ip_addresses[3].private_ip_address,
'10.0.0.24')
def run_instances(self, count, parameters, security_configured,
public_ip_needed):
"""
Spawns the specified number of EC2 instances using the parameters
provided. This method is blocking in that it waits until the
requested VMs are properly booted up. However if the requested
VMs cannot be procured within 1800 seconds, this method will treat
it as an error and return. (Also see documentation for the BaseAgent
class)
Args:
count: Number of VMs to spawned.
parameters: A dictionary of parameters. This must contain
'keyname', 'group', 'image_id' and 'instance_type' parameters.
security_configured: Uses this boolean value as an heuristic to
detect brand new AppScale deployments.
public_ip_needed: A boolean, specifies whether to launch with a public
ip or not.
Returns:
A tuple of the form (instances, public_ips, private_ips)
"""
image_id = parameters[self.PARAM_IMAGE_ID]
instance_type = parameters[self.PARAM_INSTANCE_TYPE]
keyname = parameters[self.PARAM_KEYNAME]
group = parameters[self.PARAM_GROUP]
zone = parameters[self.PARAM_ZONE]
# In case of autoscaling, the server side passes these parameters as a
# string, so this check makes sure that spot instances are only created
# when the flag is True.
spot = parameters[self.PARAM_SPOT] in ['True', 'true', True]
AppScaleLogger.log("Starting {0} machines with machine id {1}, with " \
"instance type {2}, keyname {3}, in security group {4}, in availability" \
" zone {5}".format(count, image_id, instance_type, keyname, group, zone))
if spot:
AppScaleLogger.log("Using spot instances")
else:
AppScaleLogger.log("Using on-demand instances")
start_time = datetime.datetime.now()
active_public_ips = []
active_private_ips = []
active_instances = []
# Make sure we do not have terminated instances using the same keyname.
instances = self.__describe_instances(parameters)
term_instance_info = self.__get_instance_info(instances, 'terminated',
keyname)
if len(term_instance_info[2]):
self.handle_failure('SSH keyname {0} is already registered to a '\
'terminated instance. Please change the "keyname" '\
'you specified in your AppScalefile to a different '\
'value. If the keyname was autogenerated, erase it '\
'to have a new one generated for you.'.format(keyname))
try:
attempts = 1
while True:
instance_info = self.describe_instances(parameters)
active_public_ips = instance_info[0]
active_private_ips = instance_info[1]
active_instances = instance_info[2]
# If security has been configured on this agent just now,
# that's an indication that this is a fresh cloud deployment.
# As such it's not expected to have any running VMs.
if len(active_instances) > 0 or security_configured:
break
elif attempts == self.DESCRIBE_INSTANCES_RETRY_COUNT:
self.handle_failure('Failed to invoke describe_instances')
attempts += 1
# Get subnet from parameters.
subnet = parameters.get(self.PARAM_SUBNET_ID)
network_interfaces = None
groups = None
conn = self.open_connection(parameters)
# A subnet indicates we're using VPC Networking.
if subnet:
# Get security group by name.
try:
sg = self.get_security_group_by_name(
conn, group, parameters[self.PARAM_VPC_ID])
except SecurityGroupNotFoundException as e:
raise AgentRuntimeException(e.message)
# Create network interface specification.
network_interface = NetworkInterfaceSpecification(
associate_public_ip_address=public_ip_needed,
groups=[sg.id],
subnet_id=subnet)
network_interfaces = NetworkInterfaceCollection(
network_interface)
else:
groups = [group]
if spot:
price = parameters[self.PARAM_SPOT_PRICE] or \
self.get_optimal_spot_price(conn, instance_type, zone)
conn.request_spot_instances(
str(price),
image_id,
key_name=keyname,
instance_type=instance_type,
count=count,
placement=zone,
security_groups=groups,
network_interfaces=network_interfaces)
else:
conn.run_instances(image_id,
count,
count,
key_name=keyname,
instance_type=instance_type,
placement=zone,
security_groups=groups,
network_interfaces=network_interfaces)
instance_ids = []
public_ips = []
private_ips = []
end_time = datetime.datetime.now() + datetime.timedelta(
0, self.MAX_VM_CREATION_TIME)
while datetime.datetime.now() < end_time:
AppScaleLogger.log("Waiting for your instances to start...")
public_ips, private_ips, instance_ids = self.describe_instances(
parameters)
# If we need a public ip, make sure we actually get one.
if public_ip_needed and not self.diff(public_ips, private_ips):
time.sleep(self.SLEEP_TIME)
continue
public_ips = self.diff(public_ips, active_public_ips)
private_ips = self.diff(private_ips, active_private_ips)
instance_ids = self.diff(instance_ids, active_instances)
if count == len(public_ips):
break
time.sleep(self.SLEEP_TIME)
if not public_ips:
self.handle_failure('No public IPs were able to be procured '
'within the time limit')
if len(public_ips) != count:
for index in range(0, len(public_ips)):
if public_ips[index] == '0.0.0.0':
instance_to_term = instance_ids[index]
AppScaleLogger.log('Instance {0} failed to get a public IP address'\
'and is being terminated'.format(instance_to_term))
conn.terminate_instances([instance_to_term])
end_time = datetime.datetime.now()
total_time = end_time - start_time
if spot:
AppScaleLogger.log("Started {0} spot instances in {1} seconds" \
.format(count, total_time.seconds))
else:
AppScaleLogger.log("Started {0} on-demand instances in {1} seconds" \
.format(count, total_time.seconds))
return instance_ids, public_ips, private_ips
except EC2ResponseError as exception:
self.handle_failure('EC2 response error while starting VMs: ' +
exception.error_message)
def create_instance(name, config, region, secrets, key_name, instance_data,
deploypass, loaned_to, loan_bug):
"""Creates an AMI instance with the given name and config. The config must
specify things like ami id."""
conn = get_connection(
region,
aws_access_key_id=secrets['aws_access_key_id'],
aws_secret_access_key=secrets['aws_secret_access_key'])
vpc = get_vpc(connection=conn,
aws_access_key_id=secrets['aws_access_key_id'],
aws_secret_access_key=secrets['aws_secret_access_key'])
# Make sure we don't request the same things twice
token = str(uuid.uuid4())[:16]
instance_data = instance_data.copy()
instance_data['name'] = name
instance_data['hostname'] = '{name}.{domain}'.format(
name=name, domain=config['domain'])
ami = conn.get_all_images(image_ids=[config["ami"]])[0]
bdm = None
if 'device_map' in config:
bdm = BlockDeviceMapping()
for device, device_info in config['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
# Overwrite root device size for HVM instances, since they cannot
# be resized online
if ami.virtualization_type == "hvm" and \
ami.root_device_name == device:
bd.size = ami.block_device_mapping[ami.root_device_name].size
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
ip_address = get_ip(instance_data['hostname'])
subnet_id = None
if ip_address:
s_id = get_subnet_id(vpc, ip_address)
if s_id in config['subnet_ids']:
if ip_available(conn, ip_address):
subnet_id = s_id
else:
log.warning("%s already assigned" % ip_address)
if not ip_address or not subnet_id:
ip_address = None
subnet_id = choice(config.get('subnet_ids'))
interface = NetworkInterfaceSpecification(
subnet_id=subnet_id,
private_ip_address=ip_address,
delete_on_termination=True,
groups=config.get('security_group_ids', []),
associate_public_ip_address=config.get("use_public_ip"))
interfaces = NetworkInterfaceCollection(interface)
while True:
try:
reservation = conn.run_instances(
image_id=config['ami'],
key_name=key_name,
instance_type=config['instance_type'],
block_device_map=bdm,
client_token=token,
disable_api_termination=bool(
config.get('disable_api_termination')),
network_interfaces=interfaces,
instance_profile_name=config.get("instance_profile_name"),
)
break
except boto.exception.BotoServerError:
log.exception("Cannot start an instance")
time.sleep(10)
instance = reservation.instances[0]
log.info("instance %s created, waiting to come up", instance)
# Wait for the instance to come up
while True:
try:
instance.update()
if instance.state == 'running':
break
except Exception:
log.warn("waiting for instance to come up, retrying in 10 sec...")
time.sleep(10)
instance.add_tag('Name', name)
instance.add_tag('FQDN', instance_data['hostname'])
instance.add_tag('created',
time.strftime("%Y-%m-%d %H:%M:%S %Z", time.gmtime()))
instance.add_tag('moz-type', config['type'])
if loaned_to:
instance.add_tag("moz-loaned-to", loaned_to)
if loan_bug:
instance.add_tag("moz-bug", loan_bug)
log.info("assimilating %s", instance)
instance.add_tag('moz-state', 'pending')
while True:
try:
assimilate(instance.private_ip_address, config, instance_data,
deploypass)
break
except:
log.warn("problem assimilating %s (%s), retrying in 10 sec ...",
instance_data['hostname'], instance.id)
time.sleep(10)
instance.add_tag('moz-state', 'ready')
def do_request_instance(region, moz_instance_type, price, ami, instance_config,
instance_type, availability_zone, slaveset, is_spot,
all_instances, dryrun):
name = get_available_slave_name(region,
moz_instance_type,
slaveset,
is_spot=is_spot,
all_instances=all_instances)
if not name:
log.debug("No slave name available for %s, %s, %s" %
(region, moz_instance_type, slaveset))
return False
subnet_id = get_avail_subnet(region, instance_config[region]["subnet_ids"],
availability_zone)
if not subnet_id:
log.debug("No free IP available for %s in %s", moz_instance_type,
availability_zone)
return False
fqdn = "{}.{}".format(name, instance_config[region]["domain"])
if is_spot:
log.debug("Spot request for %s (%s)", fqdn, price)
else:
log.debug("Starting %s", fqdn)
if dryrun:
log.info("Dry run. skipping")
return True
spec = NetworkInterfaceSpecification(
associate_public_ip_address=True,
subnet_id=subnet_id,
delete_on_termination=True,
groups=instance_config[region].get("security_group_ids"))
nc = NetworkInterfaceCollection(spec)
user_data = user_data_from_template(moz_instance_type, fqdn, region)
bdm = create_block_device_mapping(ami,
instance_config[region]['device_map'])
if is_spot:
rv = do_request_spot_instance(
region, price, ami.id, instance_type,
instance_config[region]["ssh_key"], user_data, bdm, nc,
instance_config[region].get("instance_profile_name"),
moz_instance_type, name, fqdn)
else:
rv = do_request_ondemand_instance(
region, price, ami.id, instance_type,
instance_config[region]["ssh_key"], user_data, bdm, nc,
instance_config[region].get("instance_profile_name"),
moz_instance_type, name, fqdn)
if rv:
template_values = dict(
region=region,
moz_instance_type=moz_instance_type,
instance_type=instance_type.replace(".", "-"),
life_cycle_type="spot" if is_spot else "ondemand",
virtualization=ami.virtualization_type,
root_device_type=ami.root_device_type,
jacuzzi_type=jacuzzi_suffix(slaveset),
)
name = "started.{region}.{moz_instance_type}.{instance_type}" \
".{life_cycle_type}.{virtualization}.{root_device_type}" \
".{jacuzzi_type}"
gr_log.add(name.format(**template_values), 1, collect=True)
return rv
def do_request_spot_instance(region, secrets, moz_instance_type, price, ami,
instance_config, cached_cert_dir, instance_type,
availability_zone, dryrun):
conn = aws_connect_to_region(region, secrets)
interface = get_available_interface(conn=conn,
moz_instance_type=moz_instance_type,
availability_zone=availability_zone)
if not interface:
raise RuntimeError("No free network interfaces left in %s" % region)
# TODO: check DNS
fqdn = interface.tags.get("FQDN")
if not fqdn:
raise RuntimeError("Skipping %s without FQDN" % interface)
log.debug("Spot request for %s (%s)", fqdn, price)
if dryrun:
log.info("Dry run. skipping")
return
spec = NetworkInterfaceSpecification(network_interface_id=interface.id)
nc = NetworkInterfaceCollection(spec)
ip = interface.private_ip_address
certs = get_puppet_certs(ip, secrets, cached_cert_dir)
user_data = """
FQDN="%(fqdn)s"
cd /var/lib/puppet/ssl || exit 1
%(certs)s
cd -
""" % dict(fqdn=fqdn, certs=certs)
if instance_config[region].get("lvm"):
user_data += """
mkdir -p /etc/lvm-init/
cat <<EOF > /etc/lvm-init/lvm-init.json
%s
EOF
/sbin/lvm-init
""" % json.dumps(instance_config[region])
bdm = BlockDeviceMapping()
for device, device_info in instance_config[region]['device_map'].items():
bd = BlockDeviceType()
if device_info.get('size'):
bd.size = device_info['size']
if ami.root_device_name == device:
ami_size = ami.block_device_mapping[device].size
if ami.virtualization_type == "hvm":
# Overwrite root device size for HVM instances, since they
# cannot be resized online
bd.size = ami_size
elif device_info.get('size'):
# make sure that size is enough for this AMI
assert ami_size <= device_info['size'], \
"Instance root device size cannot be smaller than AMI " \
"root device"
if device_info.get("delete_on_termination") is not False:
bd.delete_on_termination = True
if device_info.get("ephemeral_name"):
bd.ephemeral_name = device_info["ephemeral_name"]
bdm[device] = bd
sir = conn.request_spot_instances(
price=str(price),
image_id=ami.id,
count=1,
instance_type=instance_type,
key_name=instance_config[region]["ssh_key"],
user_data=user_data,
block_device_map=bdm,
network_interfaces=nc,
instance_profile_name=instance_config[region].get(
"instance_profile_name"),
)
sir[0].add_tag("moz-type", moz_instance_type)
from boto.ec2.networkinterface import NetworkInterfaceSpecification, NetworkInterfaceCollection
ec2_region = 'ap-southeast-1'
ami_id = 'ami-d6e7c084' # Ubuntu Server 14.04 LTS (HVM)
instance_type = 't2.micro'
subnet_id = 'subnet-xxxxxxx'
groups = ['sg-xxxxxx']
tags = {'Name':'wangfei-test','PROJECT':'test'}
key_name = 'xxxxx-test-20141218'
block_device_map = BlockDeviceMapping()
block_dev_type = BlockDeviceType(delete_on_termination=True, size=100)
block_device_map['/dev/sda1'] = block_dev_type
user_data = "sudo apt-get -y install htop"
networks = NetworkInterfaceSpecification(
subnet_id = subnet_id,
groups = groups,
associate_public_ip_address = True)
network_interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(networks)
conn = boto.ec2.connect_to_region(ec2_region)
reservation = conn.run_instances(
ami_id,
key_name = key_name,
network_interfaces = network_interfaces,
instance_type = instance_type,
min_count=1,
max_count=1,
block_device_map=block_device_map,
user_data=user_data,
)
def start_node():
start_logging()
print(" ".join(argv))
if len(argv) != 2:
print("Usage: %s <nodename>" % (argv[0], ), file=sys.stderr)
return 1
nodename = argv[1]
cc = ClusterConfiguration.from_config()
region = get_region()
ec2 = boto.ec2.connect_to_region(region)
if not ec2:
print("Could not connect to EC2 endpoint in region %r" % (region, ),
file=sys.stderr)
return 1
kw = {}
slurm_s3_root = cc.slurm_s3_root
kw['image_id'] = (cc.compute_ami if cc.compute_ami is not None else
amazon_linux_ami[region])
if cc.instance_profile is not None:
if cc.instance_profile.startswith("arn:"):
kw['instance_profile_arn'] = cc.instance_profile
else:
kw['instance_profile_name'] = cc.instance_profile
kw['key_name'] = cc.key_name
kw['instance_type'] = cc.compute_instance_type
if cc.compute_bid_price is not None:
end = time() + 24 * 60 * 60 # FIXME: Don't hardcode this.
kw['price'] = cc.compute_bid_price
kw['valid_until'] = strftime("%Y-%m-%dT%H:%M:%SZ", gmtime(end))
node_address = cc.get_address_for_nodename(nodename)
node_subnet = cc.get_subnet_for_address(node_address)
user_data = init_script % {
"region":
region,
"nodename":
nodename,
"os_packages":
" ".join(cc.compute_os_packages if cc.
compute_os_packages is not None else []),
"external_packages":
" ".join(cc.compute_external_packages if cc.
compute_external_packages is not None else []),
"slurm_ec2_conf":
cc.slurm_ec2_configuration,
"slurm_s3_root":
slurm_s3_root,
}
user_data = b64encode(user_data)
kw['user_data'] = user_data
# Map the ethernet interface to the correct IP address
eth0 = NetworkInterfaceSpecification(associate_public_ip_address=True,
delete_on_termination=True,
device_index=0,
groups=cc.security_groups,
private_ip_address=str(node_address),
subnet_id=node_subnet.id)
kw['network_interfaces'] = NetworkInterfaceCollection(eth0)
# Attach any ephemeral storage devices
block_device_map = BlockDeviceMapping()
block_device_map['/dev/xvda'] = BlockDeviceType(size=32, volume_type="gp2")
devices = cc.ephemeral_stores[cc.compute_instance_type]
for i, device in enumerate(devices):
drive = "/dev/sd" + chr(ord('b') + i)
block_device_map[drive] = BlockDeviceType(
ephemeral_name="ephemeral%d" % i)
kw['block_device_map'] = block_device_map
if cc.compute_bid_price is None:
print("run_instances: %r" % kw)
reservation = ec2.run_instances(**kw)
tags = {
'SLURMHostname': nodename,
'SLURMS3Root': slurm_s3_root,
'Name': "SLURM Computation Node %s" % nodename,
}
print("instances: %s" %
" ".join([instance.id for instance in reservation.instances]))
# create-tags can fail at times since the tag resource database is
# a bit behind EC2's actual state.
for i in xrange(10):
try:
ec2.create_tags(
[instance.id for instance in reservation.instances], tags)
break
except Exception as e:
print("Failed to tag instance: %s" % e, file=sys.stderr)
sleep(0.5 * i)
else:
print("request_spot_instances: %r" % kw, file=sys.stderr)
requests = ec2.request_spot_instances(**kw)
print("requests: %s" % " ".join([request.id for request in requests]))
return 0