def _clean_up(self, driver, delete_images=False):
""" Cleans up resources via a libcloud driver. """
log.info('Cleaning up resources')
if delete_images and len(self.images) > 0:
for image in self.images:
driver.delete_image(image)
if self.snapshot and len(self.images) == 0:
driver.destroy_volume_snapshot(self.snapshot)
self.snapshot = None
if self.util_node:
driver.destroy_node(self.util_node)
# Wait for node to be terminated
while ssh_connection_works(fedimg.AWS_UTIL_USER,
self.util_node.public_ips[0],
fedimg.AWS_KEYPATH):
sleep(10)
self.util_node = None
if self.util_volume:
# Destroy /dev/sdb or whatever
driver.destroy_volume(self.util_volume)
self.util_volume = None
if self.test_node:
driver.destroy_node(self.test_node)
self.test_node = None
def _clean_up(self, driver, delete_images=False):
""" Cleans up resources via a libcloud driver. """
log.info('Cleaning up resources')
if delete_images and len(self.images) > 0:
for image in self.images:
driver.delete_image(image)
if self.snapshot and len(self.images) == 0:
driver.destroy_volume_snapshot(self.snapshot)
self.snapshot = None
if self.util_node:
driver.destroy_node(self.util_node)
# Wait for node to be terminated
while ssh_connection_works(fedimg.AWS_UTIL_USER,
self.util_node.public_ips[0],
fedimg.AWS_KEYPATH):
sleep(10)
self.util_node = None
if self.util_volume:
# Destroy /dev/sdb or whatever
driver.destroy_volume(self.util_volume)
self.util_volume = None
if self.test_node:
driver.destroy_node(self.test_node)
self.test_node = None
def upload(self):
""" Registers the image in each EC2 region. """
log.info('EC2 upload process started')
# Get a starting utility AMI in some region to use as an origin
ami = self.util_amis[0] # Select the starting AMI to begin
self.destination = 'EC2 ({region})'.format(region=ami['region'])
fedimg.messenger.message('image.upload', self.build_name,
self.destination, 'started')
try:
# Connect to the region through the appropriate libcloud driver
cls = ami['driver']
driver = cls(fedimg.AWS_ACCESS_ID, fedimg.AWS_SECRET_KEY)
# select the desired node attributes
sizes = driver.list_sizes()
reg_size_id = 'm1.xlarge'
# check to make sure we have access to that size node
# TODO: Add try/except if for some reason the size isn't
# available?
size = [s for s in sizes if s.id == reg_size_id][0]
base_image = NodeImage(id=ami['ami'], name=None, driver=driver)
# Name the utility node
name = 'Fedimg AMI builder'
# Block device mapping for the utility node
# (Requires this second volume to write the image to for
# future registration.)
mappings = [{
'VirtualName': None, # cannot specify with Ebs
'Ebs': {
'VolumeSize': fedimg.AWS_UTIL_VOL_SIZE,
'VolumeType': self.vol_type,
'DeleteOnTermination': 'false'
},
'DeviceName': '/dev/sdb'
}]
# Read in the SSH key
with open(fedimg.AWS_PUBKEYPATH, 'rb') as f:
key_content = f.read()
# Add key to authorized keys for root user
step_1 = SSHKeyDeployment(key_content)
# Add script for deployment
# Device becomes /dev/xvdb on instance
script = "touch test" # this isn't so important for the util inst.
step_2 = ScriptDeployment(script)
# Create deployment object (will set up SSH key and run script)
msd = MultiStepDeployment([step_1, step_2])
log.info('Deploying utility instance')
while True:
try:
self.util_node = driver.deploy_node(
name=name,
image=base_image,
size=size,
ssh_username=fedimg.AWS_UTIL_USER,
ssh_alternate_usernames=[''],
ssh_key=fedimg.AWS_KEYPATH,
deploy=msd,
kernel_id=ami['aki'],
ex_metadata={'build': self.build_name},
ex_keyname=fedimg.AWS_KEYNAME,
ex_security_groups=['ssh'],
ex_ebs_optimized=True,
ex_blockdevicemappings=mappings)
except KeyPairDoesNotExistError:
# The keypair is missing from the current region.
# Let's install it and try again.
log.exception('Adding missing keypair to region')
driver.ex_import_keypair(fedimg.AWS_KEYNAME,
fedimg.AWS_PUBKEYPATH)
continue
except Exception as e:
# We might have an invalid security group, aka the 'ssh'
# security group doesn't exist in the current region. The
# reason this is caught here is because the related
# exception that prints`InvalidGroup.NotFound is, for
# some reason, a base exception.
if 'InvalidGroup.NotFound' in e.message:
log.exception('Adding missing security'
'group to region')
# Create the ssh security group
driver.ex_create_security_group('ssh', 'ssh only')
driver.ex_authorize_security_group(
'ssh', '22', '22', '0.0.0.0/0')
continue
else:
raise
break
# Wait until the utility node has SSH running
while not ssh_connection_works(fedimg.AWS_UTIL_USER,
self.util_node.public_ips[0],
fedimg.AWS_KEYPATH):
sleep(10)
log.info('Utility node started with SSH running')
# Connect to the utility node via SSH
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(self.util_node.public_ips[0],
username=fedimg.AWS_UTIL_USER,
key_filename=fedimg.AWS_KEYPATH)
# Curl the .raw.xz file down from the web, decompressing it
# and writing it to the secondary volume defined earlier by
# the block device mapping.
# curl with -L option, so we follow redirects
cmd = "sudo sh -c 'curl -L {0} | xzcat > /dev/xvdb'".format(
self.raw_url)
chan = client.get_transport().open_session()
chan.get_pty() # Request a pseudo-term to get around requiretty
log.info('Executing utility script')
# Run the above command and wait for its exit status
chan.exec_command(cmd)
status = chan.recv_exit_status()
if status != 0:
# There was a problem with the SSH command
log.error('Problem writing volume with utility instance')
data = "(no data)"
if chan.recv_ready():
data = chan.recv(1024 * 32)
fedimg.messenger.message('image.upload',
self.build_name,
self.destination,
'failed',
extra={'data': data})
raise EC2UtilityException(
"Problem writing image to utility instance volume. "
"Command exited with status {0}.\n"
"command: {1}\n"
"output: {2}".format(status, cmd, data))
client.close()
# Get volume name that image was written to
vol_id = [
x['ebs']['volume_id']
for x in self.util_node.extra['block_device_mapping']
if x['device_name'] == '/dev/sdb'
][0]
log.info('Destroying utility node')
# Terminate the utility instance
driver.destroy_node(self.util_node)
# Wait for utility node to be terminated
while ssh_connection_works(fedimg.AWS_UTIL_USER,
self.util_node.public_ips[0],
fedimg.AWS_KEYPATH):
sleep(10)
# Wait a little longer since loss of SSH connectivity doesn't mean
# that the node's destroyed
# TODO: Check instance state rather than this lame sleep thing
sleep(45)
# Take a snapshot of the volume the image was written to
self.util_volume = [
v for v in driver.list_volumes() if v.id == vol_id
][0]
snap_name = 'fedimg-snap-{0}'.format(self.build_name)
log.info('Taking a snapshot of the written volume')
self.snapshot = driver.create_volume_snapshot(self.util_volume,
name=snap_name)
snap_id = str(self.snapshot.id)
while self.snapshot.extra['state'] != 'completed':
# Re-obtain snapshot object to get updates on its state
self.snapshot = [
s for s in driver.list_snapshots() if s.id == snap_id
][0]
sleep(10)
log.info('Snapshot taken')
# Delete the volume now that we've got the snapshot
driver.destroy_volume(self.util_volume)
# make sure Fedimg knows that the vol is gone
self.util_volume = None
log.info('Destroyed volume')
# Actually register image
log.info('Registering image as an AMI')
if self.virt_type == 'paravirtual':
image_name = "{0}-{1}-PV-{2}-0".format(self.build_name,
ami['region'],
self.vol_type)
test_size_id = 'm1.xlarge'
# test_amis will include AKIs of the appropriate arch
registration_aki = [
a['aki'] for a in self.test_amis
if a['region'] == ami['region']
][0]
reg_root_device_name = '/dev/sda'
else: # HVM
image_name = "{0}-{1}-HVM-{2}-0".format(
self.build_name, ami['region'], self.vol_type)
test_size_id = 'm3.2xlarge'
# Can't supply a kernel image with HVM
registration_aki = None
reg_root_device_name = '/dev/sda1'
# For this block device mapping, we have our volume be
# based on the snapshot's ID
mapping = [{
'DeviceName': reg_root_device_name,
'Ebs': {
'SnapshotId': snap_id,
'VolumeSize': fedimg.AWS_TEST_VOL_SIZE,
'VolumeType': self.vol_type,
'DeleteOnTermination': 'true'
}
}]
# Avoid duplicate image name by incrementing the number at the
# end of the image name if there is already an AMI with that name.
# TODO: This process could be written nicer.
while True:
try:
if self.dup_count > 0:
# Remove trailing '-0' or '-1' or '-2' or...
image_name = '-'.join(image_name.split('-')[:-1])
# Re-add trailing dup number with new count
image_name += '-{0}'.format(self.dup_count)
# Try to register with that name
self.images.append(
driver.ex_register_image(
image_name,
description=self.image_desc,
root_device_name=reg_root_device_name,
block_device_mapping=mapping,
virtualization_type=self.virt_type,
kernel_id=registration_aki,
architecture=self.image_arch))
except Exception as e:
# Check if the problem was a duplicate name
if 'InvalidAMIName.Duplicate' in e.message:
# Keep trying until an unused name is found
self.dup_count += 1
continue
else:
raise
break
log.info('Completed image registration')
# Emit success fedmsg
# TODO: Can probably move this into the above try/except,
# to avoid just dumping all the messages at once.
for image in self.images:
fedimg.messenger.message('image.upload',
self.build_name,
self.destination,
'completed',
extra={
'id': image.id,
'virt_type': self.virt_type,
'vol_type': self.vol_type
})
# Now, we'll spin up a node of the AMI to test:
# Add script for deployment
# Device becomes /dev/xvdb on instance
script = "touch test"
step_2 = ScriptDeployment(script)
# Create deployment object
msd = MultiStepDeployment([step_1, step_2])
log.info('Deploying test node')
# Pick a name for the test instance
name = 'Fedimg AMI tester'
# Select the appropriate size for the instance
size = [s for s in sizes if s.id == test_size_id][0]
# Alert the fedmsg bus that an image test is starting
fedimg.messenger.message('image.test',
self.build_name,
self.destination,
'started',
extra={
'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type
})
# Actually deploy the test instance
try:
self.test_node = driver.deploy_node(
name=name,
image=self.images[0],
size=size,
ssh_username=fedimg.AWS_TEST_USER,
ssh_alternate_usernames=['root'],
ssh_key=fedimg.AWS_KEYPATH,
deploy=msd,
kernel_id=registration_aki,
ex_metadata={'build': self.build_name},
ex_keyname=fedimg.AWS_KEYNAME,
ex_security_groups=['ssh'],
)
except Exception as e:
fedimg.messenger.message('image.test',
self.build_name,
self.destination,
'failed',
extra={
'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type
})
raise EC2AMITestException("Failed to boot test node %r." % e)
# Wait until the test node has SSH running
while not ssh_connection_works(fedimg.AWS_TEST_USER,
self.test_node.public_ips[0],
fedimg.AWS_KEYPATH):
sleep(10)
log.info('Starting AMI tests')
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(self.test_node.public_ips[0],
username=fedimg.AWS_TEST_USER,
key_filename=fedimg.AWS_KEYPATH)
# Run /bin/true on the test instance as a simple "does it
# work" test
cmd = "/bin/true"
chan = client.get_transport().open_session()
chan.get_pty() # Request a pseudo-term to get around requiretty
log.info('Running AMI test script')
chan.exec_command(cmd)
# Again, wait for the test command's exit status
if chan.recv_exit_status() != 0:
# There was a problem with the SSH command
log.error('Problem testing new AMI')
data = "(no data)"
if chan.recv_ready():
data = chan.recv(1024 * 32)
fedimg.messenger.message('image.test',
self.build_name,
self.destination,
'failed',
extra={
'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type,
'data': data
})
raise EC2AMITestException("Tests on AMI failed.\n"
"output: %s" % data)
client.close()
log.info('AMI test completed')
fedimg.messenger.message('image.test',
self.build_name,
self.destination,
'completed',
extra={
'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type
})
# Let this EC2Service know that the AMI test passed, so
# it knows how to proceed.
self.test_success = True
log.info('Destroying test node')
# Destroy the test node
driver.destroy_node(self.test_node)
# Make AMIs public
for image in self.images:
driver.ex_modify_image_attribute(
image, {'LaunchPermission.Add.1.Group': 'all'})
except EC2UtilityException as e:
log.exception("Failure")
if fedimg.CLEAN_UP_ON_FAILURE:
self._clean_up(driver,
delete_images=fedimg.DELETE_IMAGES_ON_FAILURE)
return 1
except EC2AMITestException as e:
log.exception("Failure")
if fedimg.CLEAN_UP_ON_FAILURE:
self._clean_up(driver,
delete_images=fedimg.DELETE_IMAGES_ON_FAILURE)
return 1
except DeploymentException as e:
log.exception("Problem deploying node: {0}".format(e.value))
if fedimg.CLEAN_UP_ON_FAILURE:
self._clean_up(driver,
delete_images=fedimg.DELETE_IMAGES_ON_FAILURE)
return 1
except Exception as e:
# Just give a general failure message.
log.exception("Unexpected exception")
if fedimg.CLEAN_UP_ON_FAILURE:
self._clean_up(driver,
delete_images=fedimg.DELETE_IMAGES_ON_FAILURE)
return 1
else:
self._clean_up(driver)
if self.test_success:
# Copy the AMI to every other region if tests passed
copied_images = list() # completed image copies (ami: image)
# Use the AMI list as a way to cycle through the regions
for ami in self.test_amis[1:]: # we don't need the origin region
# Choose an appropriate destination name for the copy
alt_dest = 'EC2 ({region})'.format(region=ami['region'])
fedimg.messenger.message('image.upload', self.build_name,
alt_dest, 'started')
# Connect to the libcloud EC2 driver for the region we
# want to copy into
alt_cls = ami['driver']
alt_driver = alt_cls(fedimg.AWS_ACCESS_ID,
fedimg.AWS_SECRET_KEY)
# Construct the full name for the image copy
if self.virt_type == 'paravirtual':
image_name = "{0}-{1}-PV-{2}-0".format(
self.build_name, ami['region'], self.vol_type)
else: # HVM
image_name = "{0}-{1}-HVM-{2}-0".format(
self.build_name, ami['region'], self.vol_type)
log.info('AMI copy to {0} started'.format(ami['region']))
# Avoid duplicate image name by incrementing the number at the
# end of the image name if there is already an AMI with
# that name.
# TODO: Again, this could be written better
while True:
try:
if self.dup_count > 0:
# Remove trailing '-0' or '-1' or '-2' or...
image_name = '-'.join(image_name.split('-')[:-1])
# Re-add trailing dup number with new count
image_name += '-{0}'.format(self.dup_count)
# Actually run the image copy from the origin region
# to the current region.
for image in self.images:
image_copy = alt_driver.copy_image(
image,
self.test_amis[0]['region'],
name=image_name,
description=self.image_desc)
# Add the image copy to a list so we can work with
# it later.
copied_images.append(image_copy)
log.info('AMI {0} copied to AMI {1}'.format(
image, image_name))
except Exception as e:
# Check if the problem was a duplicate name
if 'InvalidAMIName.Duplicate' in e.message:
# Keep trying until an unused name is found.
# This probably won't trigger, since it seems
# like EC2 doesn't mind duplicate AMI names
# when they are being copied, only registered.
# Strange, but apprently true.
self.dup_count += 1
continue
else:
# TODO: Catch a more specific exception
log.exception('Image copy to {0} failed'.format(
ami['region']))
fedimg.messenger.message('image.upload',
self.build_name, alt_dest,
'failed')
break
# Now cycle through and make all of the copied AMIs public
# once the copy process has completed. Again, use the test
# AMI list as a way to have region and arch data:
# We don't need the origin region, since the AMI was made there:
self.test_amis = self.test_amis[1:]
for image in copied_images:
ami = self.test_amis[copied_images.index(image)]
alt_cls = ami['driver']
alt_driver = alt_cls(fedimg.AWS_ACCESS_ID,
fedimg.AWS_SECRET_KEY)
# Get an appropriate name for the region in question
alt_dest = 'EC2 ({region})'.format(region=ami['region'])
# Need to wait until the copy finishes in order to make
# the AMI public.
while True:
try:
# Make the image public
alt_driver.ex_modify_image_attribute(
image, {'LaunchPermission.Add.1.Group': 'all'})
except Exception as e:
if 'InvalidAMIID.Unavailable' in e.message:
# The copy isn't done, so wait 20 seconds
# and try again.
sleep(20)
continue
break
log.info('Made {0} public ({1}, {2}, {3})'.format(
image.id, self.build_name, self.virt_type, self.vol_type))
fedimg.messenger.message('image.upload',
self.build_name,
alt_dest,
'completed',
extra={
'id': image.id,
'virt_type': self.virt_type,
'vol_type': self.vol_type
})
return 0
def upload(self):
""" Registers the image in each EC2 region. """
log.info('EC2 upload process started')
# Get a starting utility AMI in some region to use as an origin
ami = self.util_amis[0] # Select the starting AMI to begin
self.destination = 'EC2 ({region})'.format(region=ami['region'])
fedimg.messenger.message('image.upload', self.build_name,
self.destination, 'started')
try:
# Connect to the region through the appropriate libcloud driver
cls = ami['driver']
driver = cls(fedimg.AWS_ACCESS_ID, fedimg.AWS_SECRET_KEY)
# select the desired node attributes
sizes = driver.list_sizes()
reg_size_id = 'm1.xlarge'
# check to make sure we have access to that size node
# TODO: Add try/except if for some reason the size isn't
# available?
size = [s for s in sizes if s.id == reg_size_id][0]
base_image = NodeImage(id=ami['ami'], name=None, driver=driver)
# Name the utility node
name = 'Fedimg AMI builder'
# Block device mapping for the utility node
# (Requires this second volume to write the image to for
# future registration.)
mappings = [{'VirtualName': None, # cannot specify with Ebs
'Ebs': {'VolumeSize': fedimg.AWS_UTIL_VOL_SIZE,
'VolumeType': self.vol_type,
'DeleteOnTermination': 'false'},
'DeviceName': '/dev/sdb'}]
# Read in the SSH key
with open(fedimg.AWS_PUBKEYPATH, 'rb') as f:
key_content = f.read()
# Add key to authorized keys for root user
step_1 = SSHKeyDeployment(key_content)
# Add script for deployment
# Device becomes /dev/xvdb on instance
script = "touch test" # this isn't so important for the util inst.
step_2 = ScriptDeployment(script)
# Create deployment object (will set up SSH key and run script)
msd = MultiStepDeployment([step_1, step_2])
log.info('Deploying utility instance')
while True:
try:
self.util_node = driver.deploy_node(
name=name,
image=base_image,
size=size,
ssh_username=fedimg.AWS_UTIL_USER,
ssh_alternate_usernames=[''],
ssh_key=fedimg.AWS_KEYPATH,
deploy=msd,
kernel_id=ami['aki'],
ex_metadata={'build':
self.build_name},
ex_keyname=fedimg.AWS_KEYNAME,
ex_security_groups=['ssh'],
ex_ebs_optimized=True,
ex_blockdevicemappings=mappings)
except KeyPairDoesNotExistError:
# The keypair is missing from the current region.
# Let's install it and try again.
log.exception('Adding missing keypair to region')
driver.ex_import_keypair(fedimg.AWS_KEYNAME,
fedimg.AWS_PUBKEYPATH)
continue
except Exception as e:
# We might have an invalid security group, aka the 'ssh'
# security group doesn't exist in the current region. The
# reason this is caught here is because the related
# exception that prints`InvalidGroup.NotFound is, for
# some reason, a base exception.
if 'InvalidGroup.NotFound' in e.message:
log.exception('Adding missing security'
'group to region')
# Create the ssh security group
driver.ex_create_security_group('ssh', 'ssh only')
driver.ex_authorize_security_group('ssh', '22', '22',
'0.0.0.0/0')
continue
else:
raise
break
# Wait until the utility node has SSH running
while not ssh_connection_works(fedimg.AWS_UTIL_USER,
self.util_node.public_ips[0],
fedimg.AWS_KEYPATH):
sleep(10)
log.info('Utility node started with SSH running')
# Connect to the utility node via SSH
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(self.util_node.public_ips[0],
username=fedimg.AWS_UTIL_USER,
key_filename=fedimg.AWS_KEYPATH)
# Curl the .raw.xz file down from the web, decompressing it
# and writing it to the secondary volume defined earlier by
# the block device mapping.
# curl with -L option, so we follow redirects
cmd = "sudo sh -c 'curl -L {0} | xzcat > /dev/xvdb'".format(
self.raw_url)
chan = client.get_transport().open_session()
chan.get_pty() # Request a pseudo-term to get around requiretty
log.info('Executing utility script')
# Run the above command and wait for its exit status
chan.exec_command(cmd)
status = chan.recv_exit_status()
if status != 0:
# There was a problem with the SSH command
log.error('Problem writing volume with utility instance')
data = "(no data)"
if chan.recv_ready():
data = chan.recv(1024 * 32)
fedimg.messenger.message('image.upload', self.build_name,
self.destination, 'failed',
extra={'data': data})
raise EC2UtilityException(
"Problem writing image to utility instance volume. "
"Command exited with status {0}.\n"
"command: {1}\n"
"output: {2}".format(status, cmd, data))
client.close()
# Get volume name that image was written to
vol_id = [x['ebs']['volume_id'] for x in
self.util_node.extra['block_device_mapping'] if
x['device_name'] == '/dev/sdb'][0]
log.info('Destroying utility node')
# Terminate the utility instance
driver.destroy_node(self.util_node)
# Wait for utility node to be terminated
while ssh_connection_works(fedimg.AWS_UTIL_USER,
self.util_node.public_ips[0],
fedimg.AWS_KEYPATH):
sleep(10)
# Wait a little longer since loss of SSH connectivity doesn't mean
# that the node's destroyed
# TODO: Check instance state rather than this lame sleep thing
sleep(45)
# Take a snapshot of the volume the image was written to
self.util_volume = [v for v in driver.list_volumes()
if v.id == vol_id][0]
snap_name = 'fedimg-snap-{0}'.format(self.build_name)
log.info('Taking a snapshot of the written volume')
self.snapshot = driver.create_volume_snapshot(self.util_volume,
name=snap_name)
snap_id = str(self.snapshot.id)
while self.snapshot.extra['state'] != 'completed':
# Re-obtain snapshot object to get updates on its state
self.snapshot = [s for s in driver.list_snapshots()
if s.id == snap_id][0]
sleep(10)
log.info('Snapshot taken')
# Delete the volume now that we've got the snapshot
driver.destroy_volume(self.util_volume)
# make sure Fedimg knows that the vol is gone
self.util_volume = None
log.info('Destroyed volume')
# Actually register image
log.info('Registering image as an AMI')
if self.virt_type == 'paravirtual':
image_name = "{0}-{1}-PV-{2}-0".format(self.build_name,
ami['region'],
self.vol_type)
test_size_id = 'm1.xlarge'
# test_amis will include AKIs of the appropriate arch
registration_aki = [a['aki'] for a in self.test_amis
if a['region'] == ami['region']][0]
reg_root_device_name = '/dev/sda'
else: # HVM
image_name = "{0}-{1}-HVM-{2}-0".format(self.build_name,
ami['region'],
self.vol_type)
test_size_id = 'm3.2xlarge'
# Can't supply a kernel image with HVM
registration_aki = None
reg_root_device_name = '/dev/sda1'
# For this block device mapping, we have our volume be
# based on the snapshot's ID
mapping = [{'DeviceName': reg_root_device_name,
'Ebs': {'SnapshotId': snap_id,
'VolumeSize': fedimg.AWS_TEST_VOL_SIZE,
'VolumeType': self.vol_type,
'DeleteOnTermination': 'true'}}]
# Avoid duplicate image name by incrementing the number at the
# end of the image name if there is already an AMI with that name.
# TODO: This process could be written nicer.
while True:
try:
if self.dup_count > 0:
# Remove trailing '-0' or '-1' or '-2' or...
image_name = '-'.join(image_name.split('-')[:-1])
# Re-add trailing dup number with new count
image_name += '-{0}'.format(self.dup_count)
# Try to register with that name
self.images.append(driver.ex_register_image(
image_name,
description=self.image_desc,
root_device_name=reg_root_device_name,
block_device_mapping=mapping,
virtualization_type=self.virt_type,
kernel_id=registration_aki,
architecture=self.image_arch))
except Exception as e:
# Check if the problem was a duplicate name
if 'InvalidAMIName.Duplicate' in e.message:
# Keep trying until an unused name is found
self.dup_count += 1
continue
else:
raise
break
log.info('Completed image registration')
# Emit success fedmsg
# TODO: Can probably move this into the above try/except,
# to avoid just dumping all the messages at once.
for image in self.images:
fedimg.messenger.message('image.upload', self.build_name,
self.destination, 'completed',
extra={'id': image.id,
'virt_type': self.virt_type,
'vol_type': self.vol_type})
# Now, we'll spin up a node of the AMI to test:
# Add script for deployment
# Device becomes /dev/xvdb on instance
script = "touch test"
step_2 = ScriptDeployment(script)
# Create deployment object
msd = MultiStepDeployment([step_1, step_2])
log.info('Deploying test node')
# Pick a name for the test instance
name = 'Fedimg AMI tester'
# Select the appropriate size for the instance
size = [s for s in sizes if s.id == test_size_id][0]
# Alert the fedmsg bus that an image test is starting
fedimg.messenger.message('image.test', self.build_name,
self.destination, 'started',
extra={'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type})
# Actually deploy the test instance
try:
self.test_node = driver.deploy_node(
name=name, image=self.images[0], size=size,
ssh_username=fedimg.AWS_TEST_USER,
ssh_alternate_usernames=['root'],
ssh_key=fedimg.AWS_KEYPATH,
deploy=msd,
kernel_id=registration_aki,
ex_metadata={'build': self.build_name},
ex_keyname=fedimg.AWS_KEYNAME,
ex_security_groups=['ssh'],
)
except Exception as e:
fedimg.messenger.message('image.test', self.build_name,
self.destination, 'failed',
extra={'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type})
raise EC2AMITestException("Failed to boot test node %r." % e)
# Wait until the test node has SSH running
while not ssh_connection_works(fedimg.AWS_TEST_USER,
self.test_node.public_ips[0],
fedimg.AWS_KEYPATH):
sleep(10)
log.info('Starting AMI tests')
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(self.test_node.public_ips[0],
username=fedimg.AWS_TEST_USER,
key_filename=fedimg.AWS_KEYPATH)
# Run /bin/true on the test instance as a simple "does it
# work" test
cmd = "/bin/true"
chan = client.get_transport().open_session()
chan.get_pty() # Request a pseudo-term to get around requiretty
log.info('Running AMI test script')
chan.exec_command(cmd)
# Again, wait for the test command's exit status
if chan.recv_exit_status() != 0:
# There was a problem with the SSH command
log.error('Problem testing new AMI')
data = "(no data)"
if chan.recv_ready():
data = chan.recv(1024 * 32)
fedimg.messenger.message('image.test', self.build_name,
self.destination, 'failed',
extra={'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type,
'data': data})
raise EC2AMITestException("Tests on AMI failed.\n"
"output: %s" % data)
client.close()
log.info('AMI test completed')
fedimg.messenger.message('image.test', self.build_name,
self.destination, 'completed',
extra={'id': self.images[0].id,
'virt_type': self.virt_type,
'vol_type': self.vol_type})
# Let this EC2Service know that the AMI test passed, so
# it knows how to proceed.
self.test_success = True
log.info('Destroying test node')
# Destroy the test node
driver.destroy_node(self.test_node)
# Make AMIs public
for image in self.images:
driver.ex_modify_image_attribute(
image,
{'LaunchPermission.Add.1.Group': 'all'})
except EC2UtilityException as e:
log.exception("Failure")
if fedimg.CLEAN_UP_ON_FAILURE:
self._clean_up(driver,
delete_images=fedimg.DELETE_IMAGES_ON_FAILURE)
return 1
except EC2AMITestException as e:
log.exception("Failure")
if fedimg.CLEAN_UP_ON_FAILURE:
self._clean_up(driver,
delete_images=fedimg.DELETE_IMAGES_ON_FAILURE)
return 1
except DeploymentException as e:
log.exception("Problem deploying node: {0}".format(e.value))
if fedimg.CLEAN_UP_ON_FAILURE:
self._clean_up(driver,
delete_images=fedimg.DELETE_IMAGES_ON_FAILURE)
return 1
except Exception as e:
# Just give a general failure message.
log.exception("Unexpected exception")
if fedimg.CLEAN_UP_ON_FAILURE:
self._clean_up(driver,
delete_images=fedimg.DELETE_IMAGES_ON_FAILURE)
return 1
else:
self._clean_up(driver)
if self.test_success:
# Copy the AMI to every other region if tests passed
copied_images = list() # completed image copies (ami: image)
# Use the AMI list as a way to cycle through the regions
for ami in self.test_amis[1:]: # we don't need the origin region
# Choose an appropriate destination name for the copy
alt_dest = 'EC2 ({region})'.format(
region=ami['region'])
fedimg.messenger.message('image.upload',
self.build_name,
alt_dest, 'started')
# Connect to the libcloud EC2 driver for the region we
# want to copy into
alt_cls = ami['driver']
alt_driver = alt_cls(fedimg.AWS_ACCESS_ID,
fedimg.AWS_SECRET_KEY)
# Construct the full name for the image copy
if self.virt_type == 'paravirtual':
image_name = "{0}-{1}-PV-{2}-0".format(
self.build_name, ami['region'], self.vol_type)
else: # HVM
image_name = "{0}-{1}-HVM-{2}-0".format(
self.build_name, ami['region'], self.vol_type)
log.info('AMI copy to {0} started'.format(ami['region']))
# Avoid duplicate image name by incrementing the number at the
# end of the image name if there is already an AMI with
# that name.
# TODO: Again, this could be written better
while True:
try:
if self.dup_count > 0:
# Remove trailing '-0' or '-1' or '-2' or...
image_name = '-'.join(image_name.split('-')[:-1])
# Re-add trailing dup number with new count
image_name += '-{0}'.format(self.dup_count)
# Actually run the image copy from the origin region
# to the current region.
for image in self.images:
image_copy = alt_driver.copy_image(
image,
self.test_amis[0]['region'],
name=image_name,
description=self.image_desc)
# Add the image copy to a list so we can work with
# it later.
copied_images.append(image_copy)
log.info('AMI {0} copied to AMI {1}'.format(
image, image_name))
except Exception as e:
# Check if the problem was a duplicate name
if 'InvalidAMIName.Duplicate' in e.message:
# Keep trying until an unused name is found.
# This probably won't trigger, since it seems
# like EC2 doesn't mind duplicate AMI names
# when they are being copied, only registered.
# Strange, but apprently true.
self.dup_count += 1
continue
else:
# TODO: Catch a more specific exception
log.exception(
'Image copy to {0} failed'.format(
ami['region']))
fedimg.messenger.message('image.upload',
self.build_name,
alt_dest, 'failed')
break
# Now cycle through and make all of the copied AMIs public
# once the copy process has completed. Again, use the test
# AMI list as a way to have region and arch data:
# We don't need the origin region, since the AMI was made there:
self.test_amis = self.test_amis[1:]
for image in copied_images:
ami = self.test_amis[copied_images.index(image)]
alt_cls = ami['driver']
alt_driver = alt_cls(fedimg.AWS_ACCESS_ID,
fedimg.AWS_SECRET_KEY)
# Get an appropriate name for the region in question
alt_dest = 'EC2 ({region})'.format(region=ami['region'])
# Need to wait until the copy finishes in order to make
# the AMI public.
while True:
try:
# Make the image public
alt_driver.ex_modify_image_attribute(
image,
{'LaunchPermission.Add.1.Group': 'all'})
except Exception as e:
if 'InvalidAMIID.Unavailable' in e.message:
# The copy isn't done, so wait 20 seconds
# and try again.
sleep(20)
continue
break
log.info('Made {0} public ({1}, {2}, {3})'.format(image.id,
self.build_name,
self.virt_type,
self.vol_type))
fedimg.messenger.message('image.upload',
self.build_name,
alt_dest, 'completed',
extra={'id': image.id,
'virt_type': self.virt_type,
'vol_type': self.vol_type})
return 0
