def _clean_up(self, driver, delete_images=False): """ Cleans up resources via a libcloud driver. """ log.info('Cleaning up resources') if delete_images and len(self.images) > 0: for image in self.images: driver.delete_image(image) if self.snapshot and len(self.images) == 0: driver.destroy_volume_snapshot(self.snapshot) self.snapshot = None if self.util_node: driver.destroy_node(self.util_node) # Wait for node to be terminated while ssh_connection_works(fedimg.AWS_UTIL_USER, self.util_node.public_ips[0], fedimg.AWS_KEYPATH): sleep(10) self.util_node = None if self.util_volume: # Destroy /dev/sdb or whatever driver.destroy_volume(self.util_volume) self.util_volume = None if self.test_node: driver.destroy_node(self.test_node) self.test_node = None
def upload(self): """ Registers the image in each EC2 region. """ log.info('EC2 upload process started') # Get a starting utility AMI in some region to use as an origin ami = self.util_amis[0] # Select the starting AMI to begin self.destination = 'EC2 ({region})'.format(region=ami['region']) fedimg.messenger.message('image.upload', self.build_name, self.destination, 'started') try: # Connect to the region through the appropriate libcloud driver cls = ami['driver'] driver = cls(fedimg.AWS_ACCESS_ID, fedimg.AWS_SECRET_KEY) # select the desired node attributes sizes = driver.list_sizes() reg_size_id = 'm1.xlarge' # check to make sure we have access to that size node # TODO: Add try/except if for some reason the size isn't # available? size = [s for s in sizes if s.id == reg_size_id][0] base_image = NodeImage(id=ami['ami'], name=None, driver=driver) # Name the utility node name = 'Fedimg AMI builder' # Block device mapping for the utility node # (Requires this second volume to write the image to for # future registration.) mappings = [{ 'VirtualName': None, # cannot specify with Ebs 'Ebs': { 'VolumeSize': fedimg.AWS_UTIL_VOL_SIZE, 'VolumeType': self.vol_type, 'DeleteOnTermination': 'false' }, 'DeviceName': '/dev/sdb' }] # Read in the SSH key with open(fedimg.AWS_PUBKEYPATH, 'rb') as f: key_content = f.read() # Add key to authorized keys for root user step_1 = SSHKeyDeployment(key_content) # Add script for deployment # Device becomes /dev/xvdb on instance script = "touch test" # this isn't so important for the util inst. step_2 = ScriptDeployment(script) # Create deployment object (will set up SSH key and run script) msd = MultiStepDeployment([step_1, step_2]) log.info('Deploying utility instance') while True: try: self.util_node = driver.deploy_node( name=name, image=base_image, size=size, ssh_username=fedimg.AWS_UTIL_USER, ssh_alternate_usernames=[''], ssh_key=fedimg.AWS_KEYPATH, deploy=msd, kernel_id=ami['aki'], ex_metadata={'build': self.build_name}, ex_keyname=fedimg.AWS_KEYNAME, ex_security_groups=['ssh'], ex_ebs_optimized=True, ex_blockdevicemappings=mappings) except KeyPairDoesNotExistError: # The keypair is missing from the current region. # Let's install it and try again. log.exception('Adding missing keypair to region') driver.ex_import_keypair(fedimg.AWS_KEYNAME, fedimg.AWS_PUBKEYPATH) continue except Exception as e: # We might have an invalid security group, aka the 'ssh' # security group doesn't exist in the current region. The # reason this is caught here is because the related # exception that prints`InvalidGroup.NotFound is, for # some reason, a base exception. if 'InvalidGroup.NotFound' in e.message: log.exception('Adding missing security' 'group to region') # Create the ssh security group driver.ex_create_security_group('ssh', 'ssh only') driver.ex_authorize_security_group( 'ssh', '22', '22', '0.0.0.0/0') continue else: raise break # Wait until the utility node has SSH running while not ssh_connection_works(fedimg.AWS_UTIL_USER, self.util_node.public_ips[0], fedimg.AWS_KEYPATH): sleep(10) log.info('Utility node started with SSH running') # Connect to the utility node via SSH client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(self.util_node.public_ips[0], username=fedimg.AWS_UTIL_USER, key_filename=fedimg.AWS_KEYPATH) # Curl the .raw.xz file down from the web, decompressing it # and writing it to the secondary volume defined earlier by # the block device mapping. # curl with -L option, so we follow redirects cmd = "sudo sh -c 'curl -L {0} | xzcat > /dev/xvdb'".format( self.raw_url) chan = client.get_transport().open_session() chan.get_pty() # Request a pseudo-term to get around requiretty log.info('Executing utility script') # Run the above command and wait for its exit status chan.exec_command(cmd) status = chan.recv_exit_status() if status != 0: # There was a problem with the SSH command log.error('Problem writing volume with utility instance') data = "(no data)" if chan.recv_ready(): data = chan.recv(1024 * 32) fedimg.messenger.message('image.upload', self.build_name, self.destination, 'failed', extra={'data': data}) raise EC2UtilityException( "Problem writing image to utility instance volume. " "Command exited with status {0}.\n" "command: {1}\n" "output: {2}".format(status, cmd, data)) client.close() # Get volume name that image was written to vol_id = [ x['ebs']['volume_id'] for x in self.util_node.extra['block_device_mapping'] if x['device_name'] == '/dev/sdb' ][0] log.info('Destroying utility node') # Terminate the utility instance driver.destroy_node(self.util_node) # Wait for utility node to be terminated while ssh_connection_works(fedimg.AWS_UTIL_USER, self.util_node.public_ips[0], fedimg.AWS_KEYPATH): sleep(10) # Wait a little longer since loss of SSH connectivity doesn't mean # that the node's destroyed # TODO: Check instance state rather than this lame sleep thing sleep(45) # Take a snapshot of the volume the image was written to self.util_volume = [ v for v in driver.list_volumes() if v.id == vol_id ][0] snap_name = 'fedimg-snap-{0}'.format(self.build_name) log.info('Taking a snapshot of the written volume') self.snapshot = driver.create_volume_snapshot(self.util_volume, name=snap_name) snap_id = str(self.snapshot.id) while self.snapshot.extra['state'] != 'completed': # Re-obtain snapshot object to get updates on its state self.snapshot = [ s for s in driver.list_snapshots() if s.id == snap_id ][0] sleep(10) log.info('Snapshot taken') # Delete the volume now that we've got the snapshot driver.destroy_volume(self.util_volume) # make sure Fedimg knows that the vol is gone self.util_volume = None log.info('Destroyed volume') # Actually register image log.info('Registering image as an AMI') if self.virt_type == 'paravirtual': image_name = "{0}-{1}-PV-{2}-0".format(self.build_name, ami['region'], self.vol_type) test_size_id = 'm1.xlarge' # test_amis will include AKIs of the appropriate arch registration_aki = [ a['aki'] for a in self.test_amis if a['region'] == ami['region'] ][0] reg_root_device_name = '/dev/sda' else: # HVM image_name = "{0}-{1}-HVM-{2}-0".format( self.build_name, ami['region'], self.vol_type) test_size_id = 'm3.2xlarge' # Can't supply a kernel image with HVM registration_aki = None reg_root_device_name = '/dev/sda1' # For this block device mapping, we have our volume be # based on the snapshot's ID mapping = [{ 'DeviceName': reg_root_device_name, 'Ebs': { 'SnapshotId': snap_id, 'VolumeSize': fedimg.AWS_TEST_VOL_SIZE, 'VolumeType': self.vol_type, 'DeleteOnTermination': 'true' } }] # Avoid duplicate image name by incrementing the number at the # end of the image name if there is already an AMI with that name. # TODO: This process could be written nicer. while True: try: if self.dup_count > 0: # Remove trailing '-0' or '-1' or '-2' or... image_name = '-'.join(image_name.split('-')[:-1]) # Re-add trailing dup number with new count image_name += '-{0}'.format(self.dup_count) # Try to register with that name self.images.append( driver.ex_register_image( image_name, description=self.image_desc, root_device_name=reg_root_device_name, block_device_mapping=mapping, virtualization_type=self.virt_type, kernel_id=registration_aki, architecture=self.image_arch)) except Exception as e: # Check if the problem was a duplicate name if 'InvalidAMIName.Duplicate' in e.message: # Keep trying until an unused name is found self.dup_count += 1 continue else: raise break log.info('Completed image registration') # Emit success fedmsg # TODO: Can probably move this into the above try/except, # to avoid just dumping all the messages at once. for image in self.images: fedimg.messenger.message('image.upload', self.build_name, self.destination, 'completed', extra={ 'id': image.id, 'virt_type': self.virt_type, 'vol_type': self.vol_type }) # Now, we'll spin up a node of the AMI to test: # Add script for deployment # Device becomes /dev/xvdb on instance script = "touch test" step_2 = ScriptDeployment(script) # Create deployment object msd = MultiStepDeployment([step_1, step_2]) log.info('Deploying test node') # Pick a name for the test instance name = 'Fedimg AMI tester' # Select the appropriate size for the instance size = [s for s in sizes if s.id == test_size_id][0] # Alert the fedmsg bus that an image test is starting fedimg.messenger.message('image.test', self.build_name, self.destination, 'started', extra={ 'id': self.images[0].id, 'virt_type': self.virt_type, 'vol_type': self.vol_type }) # Actually deploy the test instance try: self.test_node = driver.deploy_node( name=name, image=self.images[0], size=size, ssh_username=fedimg.AWS_TEST_USER, ssh_alternate_usernames=['root'], ssh_key=fedimg.AWS_KEYPATH, deploy=msd, kernel_id=registration_aki, ex_metadata={'build': self.build_name}, ex_keyname=fedimg.AWS_KEYNAME, ex_security_groups=['ssh'], ) except Exception as e: fedimg.messenger.message('image.test', self.build_name, self.destination, 'failed', extra={ 'id': self.images[0].id, 'virt_type': self.virt_type, 'vol_type': self.vol_type }) raise EC2AMITestException("Failed to boot test node %r." % e) # Wait until the test node has SSH running while not ssh_connection_works(fedimg.AWS_TEST_USER, self.test_node.public_ips[0], fedimg.AWS_KEYPATH): sleep(10) log.info('Starting AMI tests') client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(self.test_node.public_ips[0], username=fedimg.AWS_TEST_USER, key_filename=fedimg.AWS_KEYPATH) # Run /bin/true on the test instance as a simple "does it # work" test cmd = "/bin/true" chan = client.get_transport().open_session() chan.get_pty() # Request a pseudo-term to get around requiretty log.info('Running AMI test script') chan.exec_command(cmd) # Again, wait for the test command's exit status if chan.recv_exit_status() != 0: # There was a problem with the SSH command log.error('Problem testing new AMI') data = "(no data)" if chan.recv_ready(): data = chan.recv(1024 * 32) fedimg.messenger.message('image.test', self.build_name, self.destination, 'failed', extra={ 'id': self.images[0].id, 'virt_type': self.virt_type, 'vol_type': self.vol_type, 'data': data }) raise EC2AMITestException("Tests on AMI failed.\n" "output: %s" % data) client.close() log.info('AMI test completed') fedimg.messenger.message('image.test', self.build_name, self.destination, 'completed', extra={ 'id': self.images[0].id, 'virt_type': self.virt_type, 'vol_type': self.vol_type }) # Let this EC2Service know that the AMI test passed, so # it knows how to proceed. self.test_success = True log.info('Destroying test node') # Destroy the test node driver.destroy_node(self.test_node) # Make AMIs public for image in self.images: driver.ex_modify_image_attribute( image, {'LaunchPermission.Add.1.Group': 'all'}) except EC2UtilityException as e: log.exception("Failure") if fedimg.CLEAN_UP_ON_FAILURE: self._clean_up(driver, delete_images=fedimg.DELETE_IMAGES_ON_FAILURE) return 1 except EC2AMITestException as e: log.exception("Failure") if fedimg.CLEAN_UP_ON_FAILURE: self._clean_up(driver, delete_images=fedimg.DELETE_IMAGES_ON_FAILURE) return 1 except DeploymentException as e: log.exception("Problem deploying node: {0}".format(e.value)) if fedimg.CLEAN_UP_ON_FAILURE: self._clean_up(driver, delete_images=fedimg.DELETE_IMAGES_ON_FAILURE) return 1 except Exception as e: # Just give a general failure message. log.exception("Unexpected exception") if fedimg.CLEAN_UP_ON_FAILURE: self._clean_up(driver, delete_images=fedimg.DELETE_IMAGES_ON_FAILURE) return 1 else: self._clean_up(driver) if self.test_success: # Copy the AMI to every other region if tests passed copied_images = list() # completed image copies (ami: image) # Use the AMI list as a way to cycle through the regions for ami in self.test_amis[1:]: # we don't need the origin region # Choose an appropriate destination name for the copy alt_dest = 'EC2 ({region})'.format(region=ami['region']) fedimg.messenger.message('image.upload', self.build_name, alt_dest, 'started') # Connect to the libcloud EC2 driver for the region we # want to copy into alt_cls = ami['driver'] alt_driver = alt_cls(fedimg.AWS_ACCESS_ID, fedimg.AWS_SECRET_KEY) # Construct the full name for the image copy if self.virt_type == 'paravirtual': image_name = "{0}-{1}-PV-{2}-0".format( self.build_name, ami['region'], self.vol_type) else: # HVM image_name = "{0}-{1}-HVM-{2}-0".format( self.build_name, ami['region'], self.vol_type) log.info('AMI copy to {0} started'.format(ami['region'])) # Avoid duplicate image name by incrementing the number at the # end of the image name if there is already an AMI with # that name. # TODO: Again, this could be written better while True: try: if self.dup_count > 0: # Remove trailing '-0' or '-1' or '-2' or... image_name = '-'.join(image_name.split('-')[:-1]) # Re-add trailing dup number with new count image_name += '-{0}'.format(self.dup_count) # Actually run the image copy from the origin region # to the current region. for image in self.images: image_copy = alt_driver.copy_image( image, self.test_amis[0]['region'], name=image_name, description=self.image_desc) # Add the image copy to a list so we can work with # it later. copied_images.append(image_copy) log.info('AMI {0} copied to AMI {1}'.format( image, image_name)) except Exception as e: # Check if the problem was a duplicate name if 'InvalidAMIName.Duplicate' in e.message: # Keep trying until an unused name is found. # This probably won't trigger, since it seems # like EC2 doesn't mind duplicate AMI names # when they are being copied, only registered. # Strange, but apprently true. self.dup_count += 1 continue else: # TODO: Catch a more specific exception log.exception('Image copy to {0} failed'.format( ami['region'])) fedimg.messenger.message('image.upload', self.build_name, alt_dest, 'failed') break # Now cycle through and make all of the copied AMIs public # once the copy process has completed. Again, use the test # AMI list as a way to have region and arch data: # We don't need the origin region, since the AMI was made there: self.test_amis = self.test_amis[1:] for image in copied_images: ami = self.test_amis[copied_images.index(image)] alt_cls = ami['driver'] alt_driver = alt_cls(fedimg.AWS_ACCESS_ID, fedimg.AWS_SECRET_KEY) # Get an appropriate name for the region in question alt_dest = 'EC2 ({region})'.format(region=ami['region']) # Need to wait until the copy finishes in order to make # the AMI public. while True: try: # Make the image public alt_driver.ex_modify_image_attribute( image, {'LaunchPermission.Add.1.Group': 'all'}) except Exception as e: if 'InvalidAMIID.Unavailable' in e.message: # The copy isn't done, so wait 20 seconds # and try again. sleep(20) continue break log.info('Made {0} public ({1}, {2}, {3})'.format( image.id, self.build_name, self.virt_type, self.vol_type)) fedimg.messenger.message('image.upload', self.build_name, alt_dest, 'completed', extra={ 'id': image.id, 'virt_type': self.virt_type, 'vol_type': self.vol_type }) return 0
def upload(self): """ Registers the image in each EC2 region. """ log.info('EC2 upload process started') # Get a starting utility AMI in some region to use as an origin ami = self.util_amis[0] # Select the starting AMI to begin self.destination = 'EC2 ({region})'.format(region=ami['region']) fedimg.messenger.message('image.upload', self.build_name, self.destination, 'started') try: # Connect to the region through the appropriate libcloud driver cls = ami['driver'] driver = cls(fedimg.AWS_ACCESS_ID, fedimg.AWS_SECRET_KEY) # select the desired node attributes sizes = driver.list_sizes() reg_size_id = 'm1.xlarge' # check to make sure we have access to that size node # TODO: Add try/except if for some reason the size isn't # available? size = [s for s in sizes if s.id == reg_size_id][0] base_image = NodeImage(id=ami['ami'], name=None, driver=driver) # Name the utility node name = 'Fedimg AMI builder' # Block device mapping for the utility node # (Requires this second volume to write the image to for # future registration.) mappings = [{'VirtualName': None, # cannot specify with Ebs 'Ebs': {'VolumeSize': fedimg.AWS_UTIL_VOL_SIZE, 'VolumeType': self.vol_type, 'DeleteOnTermination': 'false'}, 'DeviceName': '/dev/sdb'}] # Read in the SSH key with open(fedimg.AWS_PUBKEYPATH, 'rb') as f: key_content = f.read() # Add key to authorized keys for root user step_1 = SSHKeyDeployment(key_content) # Add script for deployment # Device becomes /dev/xvdb on instance script = "touch test" # this isn't so important for the util inst. step_2 = ScriptDeployment(script) # Create deployment object (will set up SSH key and run script) msd = MultiStepDeployment([step_1, step_2]) log.info('Deploying utility instance') while True: try: self.util_node = driver.deploy_node( name=name, image=base_image, size=size, ssh_username=fedimg.AWS_UTIL_USER, ssh_alternate_usernames=[''], ssh_key=fedimg.AWS_KEYPATH, deploy=msd, kernel_id=ami['aki'], ex_metadata={'build': self.build_name}, ex_keyname=fedimg.AWS_KEYNAME, ex_security_groups=['ssh'], ex_ebs_optimized=True, ex_blockdevicemappings=mappings) except KeyPairDoesNotExistError: # The keypair is missing from the current region. # Let's install it and try again. log.exception('Adding missing keypair to region') driver.ex_import_keypair(fedimg.AWS_KEYNAME, fedimg.AWS_PUBKEYPATH) continue except Exception as e: # We might have an invalid security group, aka the 'ssh' # security group doesn't exist in the current region. The # reason this is caught here is because the related # exception that prints`InvalidGroup.NotFound is, for # some reason, a base exception. if 'InvalidGroup.NotFound' in e.message: log.exception('Adding missing security' 'group to region') # Create the ssh security group driver.ex_create_security_group('ssh', 'ssh only') driver.ex_authorize_security_group('ssh', '22', '22', '0.0.0.0/0') continue else: raise break # Wait until the utility node has SSH running while not ssh_connection_works(fedimg.AWS_UTIL_USER, self.util_node.public_ips[0], fedimg.AWS_KEYPATH): sleep(10) log.info('Utility node started with SSH running') # Connect to the utility node via SSH client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(self.util_node.public_ips[0], username=fedimg.AWS_UTIL_USER, key_filename=fedimg.AWS_KEYPATH) # Curl the .raw.xz file down from the web, decompressing it # and writing it to the secondary volume defined earlier by # the block device mapping. # curl with -L option, so we follow redirects cmd = "sudo sh -c 'curl -L {0} | xzcat > /dev/xvdb'".format( self.raw_url) chan = client.get_transport().open_session() chan.get_pty() # Request a pseudo-term to get around requiretty log.info('Executing utility script') # Run the above command and wait for its exit status chan.exec_command(cmd) status = chan.recv_exit_status() if status != 0: # There was a problem with the SSH command log.error('Problem writing volume with utility instance') data = "(no data)" if chan.recv_ready(): data = chan.recv(1024 * 32) fedimg.messenger.message('image.upload', self.build_name, self.destination, 'failed', extra={'data': data}) raise EC2UtilityException( "Problem writing image to utility instance volume. " "Command exited with status {0}.\n" "command: {1}\n" "output: {2}".format(status, cmd, data)) client.close() # Get volume name that image was written to vol_id = [x['ebs']['volume_id'] for x in self.util_node.extra['block_device_mapping'] if x['device_name'] == '/dev/sdb'][0] log.info('Destroying utility node') # Terminate the utility instance driver.destroy_node(self.util_node) # Wait for utility node to be terminated while ssh_connection_works(fedimg.AWS_UTIL_USER, self.util_node.public_ips[0], fedimg.AWS_KEYPATH): sleep(10) # Wait a little longer since loss of SSH connectivity doesn't mean # that the node's destroyed # TODO: Check instance state rather than this lame sleep thing sleep(45) # Take a snapshot of the volume the image was written to self.util_volume = [v for v in driver.list_volumes() if v.id == vol_id][0] snap_name = 'fedimg-snap-{0}'.format(self.build_name) log.info('Taking a snapshot of the written volume') self.snapshot = driver.create_volume_snapshot(self.util_volume, name=snap_name) snap_id = str(self.snapshot.id) while self.snapshot.extra['state'] != 'completed': # Re-obtain snapshot object to get updates on its state self.snapshot = [s for s in driver.list_snapshots() if s.id == snap_id][0] sleep(10) log.info('Snapshot taken') # Delete the volume now that we've got the snapshot driver.destroy_volume(self.util_volume) # make sure Fedimg knows that the vol is gone self.util_volume = None log.info('Destroyed volume') # Actually register image log.info('Registering image as an AMI') if self.virt_type == 'paravirtual': image_name = "{0}-{1}-PV-{2}-0".format(self.build_name, ami['region'], self.vol_type) test_size_id = 'm1.xlarge' # test_amis will include AKIs of the appropriate arch registration_aki = [a['aki'] for a in self.test_amis if a['region'] == ami['region']][0] reg_root_device_name = '/dev/sda' else: # HVM image_name = "{0}-{1}-HVM-{2}-0".format(self.build_name, ami['region'], self.vol_type) test_size_id = 'm3.2xlarge' # Can't supply a kernel image with HVM registration_aki = None reg_root_device_name = '/dev/sda1' # For this block device mapping, we have our volume be # based on the snapshot's ID mapping = [{'DeviceName': reg_root_device_name, 'Ebs': {'SnapshotId': snap_id, 'VolumeSize': fedimg.AWS_TEST_VOL_SIZE, 'VolumeType': self.vol_type, 'DeleteOnTermination': 'true'}}] # Avoid duplicate image name by incrementing the number at the # end of the image name if there is already an AMI with that name. # TODO: This process could be written nicer. while True: try: if self.dup_count > 0: # Remove trailing '-0' or '-1' or '-2' or... image_name = '-'.join(image_name.split('-')[:-1]) # Re-add trailing dup number with new count image_name += '-{0}'.format(self.dup_count) # Try to register with that name self.images.append(driver.ex_register_image( image_name, description=self.image_desc, root_device_name=reg_root_device_name, block_device_mapping=mapping, virtualization_type=self.virt_type, kernel_id=registration_aki, architecture=self.image_arch)) except Exception as e: # Check if the problem was a duplicate name if 'InvalidAMIName.Duplicate' in e.message: # Keep trying until an unused name is found self.dup_count += 1 continue else: raise break log.info('Completed image registration') # Emit success fedmsg # TODO: Can probably move this into the above try/except, # to avoid just dumping all the messages at once. for image in self.images: fedimg.messenger.message('image.upload', self.build_name, self.destination, 'completed', extra={'id': image.id, 'virt_type': self.virt_type, 'vol_type': self.vol_type}) # Now, we'll spin up a node of the AMI to test: # Add script for deployment # Device becomes /dev/xvdb on instance script = "touch test" step_2 = ScriptDeployment(script) # Create deployment object msd = MultiStepDeployment([step_1, step_2]) log.info('Deploying test node') # Pick a name for the test instance name = 'Fedimg AMI tester' # Select the appropriate size for the instance size = [s for s in sizes if s.id == test_size_id][0] # Alert the fedmsg bus that an image test is starting fedimg.messenger.message('image.test', self.build_name, self.destination, 'started', extra={'id': self.images[0].id, 'virt_type': self.virt_type, 'vol_type': self.vol_type}) # Actually deploy the test instance try: self.test_node = driver.deploy_node( name=name, image=self.images[0], size=size, ssh_username=fedimg.AWS_TEST_USER, ssh_alternate_usernames=['root'], ssh_key=fedimg.AWS_KEYPATH, deploy=msd, kernel_id=registration_aki, ex_metadata={'build': self.build_name}, ex_keyname=fedimg.AWS_KEYNAME, ex_security_groups=['ssh'], ) except Exception as e: fedimg.messenger.message('image.test', self.build_name, self.destination, 'failed', extra={'id': self.images[0].id, 'virt_type': self.virt_type, 'vol_type': self.vol_type}) raise EC2AMITestException("Failed to boot test node %r." % e) # Wait until the test node has SSH running while not ssh_connection_works(fedimg.AWS_TEST_USER, self.test_node.public_ips[0], fedimg.AWS_KEYPATH): sleep(10) log.info('Starting AMI tests') client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(self.test_node.public_ips[0], username=fedimg.AWS_TEST_USER, key_filename=fedimg.AWS_KEYPATH) # Run /bin/true on the test instance as a simple "does it # work" test cmd = "/bin/true" chan = client.get_transport().open_session() chan.get_pty() # Request a pseudo-term to get around requiretty log.info('Running AMI test script') chan.exec_command(cmd) # Again, wait for the test command's exit status if chan.recv_exit_status() != 0: # There was a problem with the SSH command log.error('Problem testing new AMI') data = "(no data)" if chan.recv_ready(): data = chan.recv(1024 * 32) fedimg.messenger.message('image.test', self.build_name, self.destination, 'failed', extra={'id': self.images[0].id, 'virt_type': self.virt_type, 'vol_type': self.vol_type, 'data': data}) raise EC2AMITestException("Tests on AMI failed.\n" "output: %s" % data) client.close() log.info('AMI test completed') fedimg.messenger.message('image.test', self.build_name, self.destination, 'completed', extra={'id': self.images[0].id, 'virt_type': self.virt_type, 'vol_type': self.vol_type}) # Let this EC2Service know that the AMI test passed, so # it knows how to proceed. self.test_success = True log.info('Destroying test node') # Destroy the test node driver.destroy_node(self.test_node) # Make AMIs public for image in self.images: driver.ex_modify_image_attribute( image, {'LaunchPermission.Add.1.Group': 'all'}) except EC2UtilityException as e: log.exception("Failure") if fedimg.CLEAN_UP_ON_FAILURE: self._clean_up(driver, delete_images=fedimg.DELETE_IMAGES_ON_FAILURE) return 1 except EC2AMITestException as e: log.exception("Failure") if fedimg.CLEAN_UP_ON_FAILURE: self._clean_up(driver, delete_images=fedimg.DELETE_IMAGES_ON_FAILURE) return 1 except DeploymentException as e: log.exception("Problem deploying node: {0}".format(e.value)) if fedimg.CLEAN_UP_ON_FAILURE: self._clean_up(driver, delete_images=fedimg.DELETE_IMAGES_ON_FAILURE) return 1 except Exception as e: # Just give a general failure message. log.exception("Unexpected exception") if fedimg.CLEAN_UP_ON_FAILURE: self._clean_up(driver, delete_images=fedimg.DELETE_IMAGES_ON_FAILURE) return 1 else: self._clean_up(driver) if self.test_success: # Copy the AMI to every other region if tests passed copied_images = list() # completed image copies (ami: image) # Use the AMI list as a way to cycle through the regions for ami in self.test_amis[1:]: # we don't need the origin region # Choose an appropriate destination name for the copy alt_dest = 'EC2 ({region})'.format( region=ami['region']) fedimg.messenger.message('image.upload', self.build_name, alt_dest, 'started') # Connect to the libcloud EC2 driver for the region we # want to copy into alt_cls = ami['driver'] alt_driver = alt_cls(fedimg.AWS_ACCESS_ID, fedimg.AWS_SECRET_KEY) # Construct the full name for the image copy if self.virt_type == 'paravirtual': image_name = "{0}-{1}-PV-{2}-0".format( self.build_name, ami['region'], self.vol_type) else: # HVM image_name = "{0}-{1}-HVM-{2}-0".format( self.build_name, ami['region'], self.vol_type) log.info('AMI copy to {0} started'.format(ami['region'])) # Avoid duplicate image name by incrementing the number at the # end of the image name if there is already an AMI with # that name. # TODO: Again, this could be written better while True: try: if self.dup_count > 0: # Remove trailing '-0' or '-1' or '-2' or... image_name = '-'.join(image_name.split('-')[:-1]) # Re-add trailing dup number with new count image_name += '-{0}'.format(self.dup_count) # Actually run the image copy from the origin region # to the current region. for image in self.images: image_copy = alt_driver.copy_image( image, self.test_amis[0]['region'], name=image_name, description=self.image_desc) # Add the image copy to a list so we can work with # it later. copied_images.append(image_copy) log.info('AMI {0} copied to AMI {1}'.format( image, image_name)) except Exception as e: # Check if the problem was a duplicate name if 'InvalidAMIName.Duplicate' in e.message: # Keep trying until an unused name is found. # This probably won't trigger, since it seems # like EC2 doesn't mind duplicate AMI names # when they are being copied, only registered. # Strange, but apprently true. self.dup_count += 1 continue else: # TODO: Catch a more specific exception log.exception( 'Image copy to {0} failed'.format( ami['region'])) fedimg.messenger.message('image.upload', self.build_name, alt_dest, 'failed') break # Now cycle through and make all of the copied AMIs public # once the copy process has completed. Again, use the test # AMI list as a way to have region and arch data: # We don't need the origin region, since the AMI was made there: self.test_amis = self.test_amis[1:] for image in copied_images: ami = self.test_amis[copied_images.index(image)] alt_cls = ami['driver'] alt_driver = alt_cls(fedimg.AWS_ACCESS_ID, fedimg.AWS_SECRET_KEY) # Get an appropriate name for the region in question alt_dest = 'EC2 ({region})'.format(region=ami['region']) # Need to wait until the copy finishes in order to make # the AMI public. while True: try: # Make the image public alt_driver.ex_modify_image_attribute( image, {'LaunchPermission.Add.1.Group': 'all'}) except Exception as e: if 'InvalidAMIID.Unavailable' in e.message: # The copy isn't done, so wait 20 seconds # and try again. sleep(20) continue break log.info('Made {0} public ({1}, {2}, {3})'.format(image.id, self.build_name, self.virt_type, self.vol_type)) fedimg.messenger.message('image.upload', self.build_name, alt_dest, 'completed', extra={'id': image.id, 'virt_type': self.virt_type, 'vol_type': self.vol_type}) return 0