def _register_image(self, snapshot): counter = 0 block_device_map = self._create_block_device_map(snapshot) root_device_name = self._determine_root_device_name() while True: if counter > 0: self.image_name = re.sub('\d(?!\d)$', lambda x: str(int(x.group(0))+1), self.image_name) try: _log.info('Registering the image in %r (snapshot id: %r) with ' 'name %r' % (self.region, snapshot.id, self.image_name)) image = self._connect().ex_register_image( name=self.image_name, description=self.image_description, virtualization_type=self.image_virtualization_type, architecture=self.image_architecture, block_device_mapping=block_device_map, root_device_name=root_device_name, ena_support=True ) if self.push_notifications: fedimg.messenger.notify( topic='image.upload', msg=dict( image_url=self.image_url, image_name=get_image_name_from_ami_name_for_fedmsg(self.image_name), destination=self.region, service=self.service, status='completed', compose=self.compose_id, extra=dict( id=image.id, virt_type=self.image_virtualization_type, vol_type=self.image_volume_type ) ) ) return image except Exception as e: _log.info('Could not register with name: %r' % self.image_name) if 'InvalidAMIName.Duplicate' in str(e): counter = counter + 1 else: raise
def _register_image(self, snapshot): counter = 0 block_device_map = self._create_block_device_map(snapshot) root_device_name = self._determine_root_device_name() while True: if counter > 0: self.image_name = re.sub('\d(?!\d)$', lambda x: str(int(x.group(0)) + 1), self.image_name) try: _log.info('Registering the image in %r (snapshot id: %r) with ' 'name %r' % (self.region, snapshot.id, self.image_name)) image = self._connect().ex_register_image( name=self.image_name, description=self.image_description, virtualization_type=self.image_virtualization_type, architecture=self.image_architecture, block_device_mapping=block_device_map, root_device_name=root_device_name, ena_support=True) if self.push_notifications: fedimg.messenger.notify( topic='image.upload', msg=dict( image_url=self.image_url, image_name=get_image_name_from_ami_name_for_fedmsg( self.image_name), destination=self.region, service=self.service, status='completed', compose=self.compose_id, extra=dict( id=image.id, virt_type=self.image_virtualization_type, vol_type=self.image_volume_type))) return image except Exception as e: _log.info('Could not register with name: %r' % self.image_name) if 'InvalidAMIName.Duplicate' in str(e): counter = counter + 1 else: raise
def copy_images_to_regions(self, image_id=None, base_region=None, regions=None): """ Comment goes here """ if (image_id is None) or (regions is None) or (base_region is None): return counter = 0 copied_images = [] self.set_region(base_region) image = self._connect().get_image(image_id=image_id) if not image: return [] for region in regions: _log.info('Copy %s to %s started' % (image_id, region)) self.set_region(region) self.image_name = get_image_name_from_ami_name(image.name, region) while True: if counter > 0: self.image_name = re.sub( '\d(?!\d)', lambda x: str(int(x.group(0)) + 1), self.image_name) try: copied_image = self._connect().copy_image( source_region=base_region, image=image, name=self.image_name, description=self.image_description) virt_type = image.extra['virtualization_type'] volume_type = image.extra['block_device_mapping'][0][ 'ebs']['volume_type'] if self.push_notifications: fedimg.messenger.notify( topic='image.copy', msg=dict(image_name= get_image_name_from_ami_name_for_fedmsg( copied_image.name), destination=self.region, service=self.service, compose_id=self.compose_id, extra=dict(id=copied_image.id, virt_type=virt_type, vol_type=volume_type, source_image_id=image.id))) _log.info('Copy %s to %s is completed.' % (image_id, region)) copied_images.append({ 'region': region, 'copied_image_id': copied_image.id }) break except Exception as e: _log.info('Could not register ' 'with name: %r' % self.image_name) if 'InvalidAMIName.Duplicate' in str(e): counter = counter + 1 else: _log.info('Failed') break return copied_images
def publish_images(self, region_image_mapping=None): """ Comment goes here """ published_images = [] if region_image_mapping is None: return published_images for region, image_id in region_image_mapping: self.set_region(region) _log.info('Publish image (%s) in %s started' % (image_id, region)) image = self._connect().get_image(image_id=image_id) is_image_public = self._retry_till_image_is_public(image) _log.info('Publish image (%s) in %s completed' % (image_id, region)) _log.info('Publish snaphsot for image (%s) in %s started' % (image_id, region)) snapshot = self.get_snapshot_from_image(image) _log.info('Fetched snapshot for image (%s): %s' % (image_id, snapshot.id)) is_snapshot_public = self._retry_till_snapshot_is_public(snapshot) _log.info('Publish snaphsot for image (%s) in %s completed' % (image_id, region)) volume_type = self.get_volume_type_from_image(image) virt_type = self.get_virt_type_from_image(image) if self.push_notifications: fedimg.messenger.notify( topic='image.publish', msg=dict( image_name=get_image_name_from_ami_name_for_fedmsg( image.name), image_url=self.image_url, destination=self.region, service=self.service, compose=self.compose_id, extra=dict(id=image.id, virt_type=virt_type, vol_type=volume_type))) fedimg.messenger.notify( topic='image.upload', msg=dict( image_name=get_image_name_from_ami_name_for_fedmsg( image.name), image_url=self.image_url, destination=self.region, service=self.service, status='completed', compose=self.compose_id, extra=dict(id=image.id, virt_type=virt_type, vol_type=volume_type))) published_images.append({ 'image_id': image.id, 'is_image_public': is_image_public, 'snapshot_id': snapshot.id, 'is_snapshot_public': is_snapshot_public, 'regions': self.region }) return published_images
def main(image_urls, access_id, secret_key, regions, volume_types=None, volume_via_s3=True, ex_virt_types=None, push_notifications=False, compose_id=None): """ The `ec2.ec2initiate.main` function iterates over the image urls and start uploading the image to the specified regions. The `image_urls`, `access_id`, and `regions` are the required params. `volume_types`, `ex_virt_types` are optional arguments, if not passed the values are picked up from the fedimg configuration. `volume_via_s3`, `push_notifications`, and `compose_id` are optional params with default values. Args: image_urls (list): List of the image urls to create AMIs. (reques access_id (str): AWS EC2 access id secret_key (str): AWS_EC2 secret key regions (list): List of AWS regions the AMI to be uploaded. volume_types (list): List of supported volumes for the AMIs to be created. volume_via_s3 (bool): If `True`, the images are uploaded via s3 method else using creating builder instances. ex_virt_types (list): List of the supported virts for the AMIs to be created. push_notifications (bool): If `True` the messages will be pushed to fedmsg, else skipped. compose_id: id of the current compose in process. """ root_volume_size = AWS_ROOT_VOLUME_SIZE published_images = [] if volume_types is None: volume_types = AWS_VOLUME_TYPES if regions is None: regions = [AWS_BASE_REGION] for image_url in image_urls: # If the virt types is not provided then select the supported virt # types from the image. if ex_virt_types is None: virt_types = get_virt_types_from_url(image_url) else: virt_types = ex_virt_types try: source = get_source_from_image(image_url) if not source: raise ValueError image_architecture = get_file_arch(image_url) uploader = EC2ImageUploader(compose_id=compose_id, access_key=access_id, secret_key=secret_key, root_volume_size=root_volume_size, image_architecture=image_architecture, volume_via_s3=volume_via_s3, push_notifications=push_notifications, image_url=image_url) publisher = EC2ImagePublisher( compose_id=compose_id, access_key=access_id, secret_key=secret_key, push_notifications=push_notifications, image_url=image_url) combinations = itertools_product( *[regions, virt_types, volume_types]) for region, virt_type, volume_type in combinations: uploader.set_region(region) _log.debug('(uploader) Region is set to: %r' % region) uploader.set_image_virt_type(virt_type) _log.debug('(uploader) Virtualization type ' 'is set to: %r' % virt_type) image_name = get_image_name_from_image(image_url=image_url, virt_type=virt_type, region=region, volume_type=volume_type) uploader.set_image_name(image_name) uploader.set_image_volume_type(volume_type) _log.debug('(uploader) Volume type is set to: %r' % volume_type) uploader.set_availability_zone_for_region() if push_notifications: fedimg.messenger.notify( topic='image.upload', msg=dict( image_url=image_url, image_name=get_image_name_from_ami_name_for_fedmsg( image_name), destination=region, service='EC2', status='started', compose=compose_id, extra=dict(virt_type=virt_type, vol_type=volume_type))) image = uploader.create_image(source) published_images.extend( publisher.publish_images( region_image_mapping=[(region, image.id)])) except Exception as e: _log.debug(e.message) #TODO: Implement the clean up of the images if failed. # uploader.clean_up(image_id=image.id, delete_snapshot=True) shutil.rmtree(os.path.dirname(source)) return published_images
def copy_images_to_regions(self, image_id=None, base_region=None, regions=None): """ Comment goes here """ if (image_id is None) or (regions is None) or (base_region is None): return counter = 0 copied_images = [] self.set_region(base_region) image = self._connect().get_image(image_id=image_id) if not image: return [] for region in regions: _log.info('Copy %s to %s started' % (image_id, region)) self.set_region(region) self.image_name = get_image_name_from_ami_name(image.name, region) while True: if counter > 0: self.image_name = re.sub( '\d(?!\d)', lambda x: str(int(x.group(0))+1), self.image_name ) try: copied_image = self._connect().copy_image( source_region=base_region, image=image, name=self.image_name, description=self.image_description) copied_image = self._retry_till_image_is_available(copied_image.id) virt_type = image.extra['virtualization_type'] volume_type = image.extra['block_device_mapping'][0]['ebs']['volume_type'] if self.push_notifications: fedimg.messenger.notify( topic='image.copy', msg=dict( image_name=get_image_name_from_ami_name_for_fedmsg(copied_image.name), destination=self.region, service=self.service, compose_id=self.compose_id, extra=dict( id=copied_image.id, virt_type=virt_type, vol_type=volume_type, source_image_id=image.id ) ) ) _log.info('Copy %s to %s is completed.' % (image_id, region)) copied_images.append({ 'region': region, 'copied_image_id': copied_image.id }) break except Exception as e: _log.info('Could not register ' 'with name: %r' % self.image_name) if 'InvalidAMIName.Duplicate' in str(e): counter = counter + 1 else: _log.info('Failed') break return copied_images
def publish_images(self, region_image_mapping=None): """ Comment goes here """ published_images = [] if region_image_mapping is None: return published_images for region, image_id in region_image_mapping: self.set_region(region) _log.info('Publish image (%s) in %s started' % (image_id, region)) image = self._connect().get_image(image_id=image_id) is_image_public = self._retry_till_image_is_public(image) _log.info('Publish image (%s) in %s completed' % (image_id, region)) _log.info('Publish snaphsot for image (%s) in %s started' % (image_id, region)) snapshot = self.get_snapshot_from_image(image) _log.info('Fetched snapshot for image (%s): %s' % (image_id, snapshot.id)) is_snapshot_public = self._retry_till_snapshot_is_public(snapshot) _log.info('Publish snaphsot for image (%s) in %s completed' % (image_id, region)) volume_type = self.get_volume_type_from_image(image) virt_type = self.get_virt_type_from_image(image) if self.push_notifications: fedimg.messenger.notify( topic='image.publish', msg=dict( image_name=get_image_name_from_ami_name_for_fedmsg(image.name), image_url=self.image_url, destination=self.region, service=self.service, compose=self.compose_id, extra=dict( id=image.id, virt_type=virt_type, vol_type=volume_type ) ) ) fedimg.messenger.notify( topic='image.upload', msg=dict( image_name=get_image_name_from_ami_name_for_fedmsg(image.name), image_url=self.image_url, destination=self.region, service=self.service, status='completed', compose=self.compose_id, extra=dict( id=image.id, virt_type=virt_type, vol_type=volume_type ) ) ) published_images.append({ 'image_id': image.id, 'is_image_public': is_image_public, 'snapshot_id': snapshot.id, 'is_snapshot_public': is_snapshot_public, 'regions': self.region }) return published_images