def _register_image(self, snapshot):
counter = 0
block_device_map = self._create_block_device_map(snapshot)
root_device_name = self._determine_root_device_name()
while True:
if counter > 0:
self.image_name = re.sub('\d(?!\d)$',
lambda x: str(int(x.group(0))+1),
self.image_name)
try:
_log.info('Registering the image in %r (snapshot id: %r) with '
'name %r' % (self.region, snapshot.id,
self.image_name))
image = self._connect().ex_register_image(
name=self.image_name,
description=self.image_description,
virtualization_type=self.image_virtualization_type,
architecture=self.image_architecture,
block_device_mapping=block_device_map,
root_device_name=root_device_name,
ena_support=True
)
if self.push_notifications:
fedimg.messenger.notify(
topic='image.upload',
msg=dict(
image_url=self.image_url,
image_name=get_image_name_from_ami_name_for_fedmsg(self.image_name),
destination=self.region,
service=self.service,
status='completed',
compose=self.compose_id,
extra=dict(
id=image.id,
virt_type=self.image_virtualization_type,
vol_type=self.image_volume_type
)
)
)
return image
except Exception as e:
_log.info('Could not register with name: %r' % self.image_name)
if 'InvalidAMIName.Duplicate' in str(e):
counter = counter + 1
else:
raise
def _register_image(self, snapshot):
counter = 0
block_device_map = self._create_block_device_map(snapshot)
root_device_name = self._determine_root_device_name()
while True:
if counter > 0:
self.image_name = re.sub('\d(?!\d)$',
lambda x: str(int(x.group(0)) + 1),
self.image_name)
try:
_log.info('Registering the image in %r (snapshot id: %r) with '
'name %r' %
(self.region, snapshot.id, self.image_name))
image = self._connect().ex_register_image(
name=self.image_name,
description=self.image_description,
virtualization_type=self.image_virtualization_type,
architecture=self.image_architecture,
block_device_mapping=block_device_map,
root_device_name=root_device_name,
ena_support=True)
if self.push_notifications:
fedimg.messenger.notify(
topic='image.upload',
msg=dict(
image_url=self.image_url,
image_name=get_image_name_from_ami_name_for_fedmsg(
self.image_name),
destination=self.region,
service=self.service,
status='completed',
compose=self.compose_id,
extra=dict(
id=image.id,
virt_type=self.image_virtualization_type,
vol_type=self.image_volume_type)))
return image
except Exception as e:
_log.info('Could not register with name: %r' % self.image_name)
if 'InvalidAMIName.Duplicate' in str(e):
counter = counter + 1
else:
raise
def copy_images_to_regions(self,
image_id=None,
base_region=None,
regions=None):
""" Comment goes here """
if (image_id is None) or (regions is None) or (base_region is None):
return
counter = 0
copied_images = []
self.set_region(base_region)
image = self._connect().get_image(image_id=image_id)
if not image:
return []
for region in regions:
_log.info('Copy %s to %s started' % (image_id, region))
self.set_region(region)
self.image_name = get_image_name_from_ami_name(image.name, region)
while True:
if counter > 0:
self.image_name = re.sub(
'\d(?!\d)', lambda x: str(int(x.group(0)) + 1),
self.image_name)
try:
copied_image = self._connect().copy_image(
source_region=base_region,
image=image,
name=self.image_name,
description=self.image_description)
virt_type = image.extra['virtualization_type']
volume_type = image.extra['block_device_mapping'][0][
'ebs']['volume_type']
if self.push_notifications:
fedimg.messenger.notify(
topic='image.copy',
msg=dict(image_name=
get_image_name_from_ami_name_for_fedmsg(
copied_image.name),
destination=self.region,
service=self.service,
compose_id=self.compose_id,
extra=dict(id=copied_image.id,
virt_type=virt_type,
vol_type=volume_type,
source_image_id=image.id)))
_log.info('Copy %s to %s is completed.' %
(image_id, region))
copied_images.append({
'region': region,
'copied_image_id': copied_image.id
})
break
except Exception as e:
_log.info('Could not register '
'with name: %r' % self.image_name)
if 'InvalidAMIName.Duplicate' in str(e):
counter = counter + 1
else:
_log.info('Failed')
break
return copied_images
def publish_images(self, region_image_mapping=None):
""" Comment goes here """
published_images = []
if region_image_mapping is None:
return published_images
for region, image_id in region_image_mapping:
self.set_region(region)
_log.info('Publish image (%s) in %s started' % (image_id, region))
image = self._connect().get_image(image_id=image_id)
is_image_public = self._retry_till_image_is_public(image)
_log.info('Publish image (%s) in %s completed' %
(image_id, region))
_log.info('Publish snaphsot for image (%s) in %s started' %
(image_id, region))
snapshot = self.get_snapshot_from_image(image)
_log.info('Fetched snapshot for image (%s): %s' %
(image_id, snapshot.id))
is_snapshot_public = self._retry_till_snapshot_is_public(snapshot)
_log.info('Publish snaphsot for image (%s) in %s completed' %
(image_id, region))
volume_type = self.get_volume_type_from_image(image)
virt_type = self.get_virt_type_from_image(image)
if self.push_notifications:
fedimg.messenger.notify(
topic='image.publish',
msg=dict(
image_name=get_image_name_from_ami_name_for_fedmsg(
image.name),
image_url=self.image_url,
destination=self.region,
service=self.service,
compose=self.compose_id,
extra=dict(id=image.id,
virt_type=virt_type,
vol_type=volume_type)))
fedimg.messenger.notify(
topic='image.upload',
msg=dict(
image_name=get_image_name_from_ami_name_for_fedmsg(
image.name),
image_url=self.image_url,
destination=self.region,
service=self.service,
status='completed',
compose=self.compose_id,
extra=dict(id=image.id,
virt_type=virt_type,
vol_type=volume_type)))
published_images.append({
'image_id': image.id,
'is_image_public': is_image_public,
'snapshot_id': snapshot.id,
'is_snapshot_public': is_snapshot_public,
'regions': self.region
})
return published_images
def main(image_urls,
access_id,
secret_key,
regions,
volume_types=None,
volume_via_s3=True,
ex_virt_types=None,
push_notifications=False,
compose_id=None):
"""
The `ec2.ec2initiate.main` function iterates over the image urls and start
uploading the image to the specified regions. The `image_urls`,
`access_id`, and `regions` are the required params. `volume_types`,
`ex_virt_types` are optional arguments, if not passed the values are picked
up from the fedimg configuration.
`volume_via_s3`, `push_notifications`, and `compose_id` are optional params
with default values.
Args:
image_urls (list): List of the image urls to create AMIs. (reques
access_id (str): AWS EC2 access id
secret_key (str): AWS_EC2 secret key
regions (list): List of AWS regions the AMI to be uploaded.
volume_types (list): List of supported volumes for the AMIs to
be created.
volume_via_s3 (bool): If `True`, the images are uploaded via s3 method
else using creating builder instances.
ex_virt_types (list): List of the supported virts for the AMIs to
be created.
push_notifications (bool): If `True` the messages will be pushed to
fedmsg, else skipped.
compose_id: id of the current compose in process.
"""
root_volume_size = AWS_ROOT_VOLUME_SIZE
published_images = []
if volume_types is None:
volume_types = AWS_VOLUME_TYPES
if regions is None:
regions = [AWS_BASE_REGION]
for image_url in image_urls:
# If the virt types is not provided then select the supported virt
# types from the image.
if ex_virt_types is None:
virt_types = get_virt_types_from_url(image_url)
else:
virt_types = ex_virt_types
try:
source = get_source_from_image(image_url)
if not source:
raise ValueError
image_architecture = get_file_arch(image_url)
uploader = EC2ImageUploader(compose_id=compose_id,
access_key=access_id,
secret_key=secret_key,
root_volume_size=root_volume_size,
image_architecture=image_architecture,
volume_via_s3=volume_via_s3,
push_notifications=push_notifications,
image_url=image_url)
publisher = EC2ImagePublisher(
compose_id=compose_id,
access_key=access_id,
secret_key=secret_key,
push_notifications=push_notifications,
image_url=image_url)
combinations = itertools_product(
*[regions, virt_types, volume_types])
for region, virt_type, volume_type in combinations:
uploader.set_region(region)
_log.debug('(uploader) Region is set to: %r' % region)
uploader.set_image_virt_type(virt_type)
_log.debug('(uploader) Virtualization type '
'is set to: %r' % virt_type)
image_name = get_image_name_from_image(image_url=image_url,
virt_type=virt_type,
region=region,
volume_type=volume_type)
uploader.set_image_name(image_name)
uploader.set_image_volume_type(volume_type)
_log.debug('(uploader) Volume type is set to: %r' %
volume_type)
uploader.set_availability_zone_for_region()
if push_notifications:
fedimg.messenger.notify(
topic='image.upload',
msg=dict(
image_url=image_url,
image_name=get_image_name_from_ami_name_for_fedmsg(
image_name),
destination=region,
service='EC2',
status='started',
compose=compose_id,
extra=dict(virt_type=virt_type,
vol_type=volume_type)))
image = uploader.create_image(source)
published_images.extend(
publisher.publish_images(
region_image_mapping=[(region, image.id)]))
except Exception as e:
_log.debug(e.message)
#TODO: Implement the clean up of the images if failed.
# uploader.clean_up(image_id=image.id, delete_snapshot=True)
shutil.rmtree(os.path.dirname(source))
return published_images
def copy_images_to_regions(self, image_id=None, base_region=None, regions=None):
""" Comment goes here """
if (image_id is None) or (regions is None) or (base_region is None):
return
counter = 0
copied_images = []
self.set_region(base_region)
image = self._connect().get_image(image_id=image_id)
if not image:
return []
for region in regions:
_log.info('Copy %s to %s started' % (image_id, region))
self.set_region(region)
self.image_name = get_image_name_from_ami_name(image.name, region)
while True:
if counter > 0:
self.image_name = re.sub(
'\d(?!\d)',
lambda x: str(int(x.group(0))+1),
self.image_name
)
try:
copied_image = self._connect().copy_image(
source_region=base_region,
image=image,
name=self.image_name,
description=self.image_description)
copied_image = self._retry_till_image_is_available(copied_image.id)
virt_type = image.extra['virtualization_type']
volume_type = image.extra['block_device_mapping'][0]['ebs']['volume_type']
if self.push_notifications:
fedimg.messenger.notify(
topic='image.copy',
msg=dict(
image_name=get_image_name_from_ami_name_for_fedmsg(copied_image.name),
destination=self.region,
service=self.service,
compose_id=self.compose_id,
extra=dict(
id=copied_image.id,
virt_type=virt_type,
vol_type=volume_type,
source_image_id=image.id
)
)
)
_log.info('Copy %s to %s is completed.' % (image_id, region))
copied_images.append({
'region': region,
'copied_image_id': copied_image.id
})
break
except Exception as e:
_log.info('Could not register '
'with name: %r' % self.image_name)
if 'InvalidAMIName.Duplicate' in str(e):
counter = counter + 1
else:
_log.info('Failed')
break
return copied_images
def publish_images(self, region_image_mapping=None):
""" Comment goes here """
published_images = []
if region_image_mapping is None:
return published_images
for region, image_id in region_image_mapping:
self.set_region(region)
_log.info('Publish image (%s) in %s started' % (image_id, region))
image = self._connect().get_image(image_id=image_id)
is_image_public = self._retry_till_image_is_public(image)
_log.info('Publish image (%s) in %s completed' % (image_id, region))
_log.info('Publish snaphsot for image (%s) in %s started' % (image_id, region))
snapshot = self.get_snapshot_from_image(image)
_log.info('Fetched snapshot for image (%s): %s' % (image_id, snapshot.id))
is_snapshot_public = self._retry_till_snapshot_is_public(snapshot)
_log.info('Publish snaphsot for image (%s) in %s completed' % (image_id, region))
volume_type = self.get_volume_type_from_image(image)
virt_type = self.get_virt_type_from_image(image)
if self.push_notifications:
fedimg.messenger.notify(
topic='image.publish',
msg=dict(
image_name=get_image_name_from_ami_name_for_fedmsg(image.name),
image_url=self.image_url,
destination=self.region,
service=self.service,
compose=self.compose_id,
extra=dict(
id=image.id,
virt_type=virt_type,
vol_type=volume_type
)
)
)
fedimg.messenger.notify(
topic='image.upload',
msg=dict(
image_name=get_image_name_from_ami_name_for_fedmsg(image.name),
image_url=self.image_url,
destination=self.region,
service=self.service,
status='completed',
compose=self.compose_id,
extra=dict(
id=image.id,
virt_type=virt_type,
vol_type=volume_type
)
)
)
published_images.append({
'image_id': image.id,
'is_image_public': is_image_public,
'snapshot_id': snapshot.id,
'is_snapshot_public': is_snapshot_public,
'regions': self.region
})
return published_images