def export_sync_region_status(self, regionstr, stream): """export a csv report about the images pending to sync in this region The report follow this pattern: <regionname>,<status>,<imagename> Status can be: *ok: the image is synchronised *ok_stalled_checksum: the image is different (has other checksum) than the master image, but users specifically has asked don't update this image. *pending_upload: the image is not synchronised *pending_metadata: the image is uploaded, but some metadata must be updated. *pending_replace: the image must be replaced, because the checksum is different. *pending_rename: the image must be replaced, but before this the old image will be renamed. *pending_ami: the image has kernel_id or ramdisk_id and this value is pending because the image has not been uploaded yet. *error_ami: the image references a kernel_id o ramdisk_id that is not included in the set of images to synchronise *error_checksum: the image has different checksum than master and there is no information about what to do (dontupdate, replace, rename) :param regionstr: A region specified as 'target:region'. The prefix 'master:' may be omitted. :param stream: Stream object (e.g. a file) where the data is written :return: Nothing """ regionobj = GlanceSyncRegion(regionstr, self.targets) target = regionobj.target target['tenant_id'] = target['facade'].get_tenant_id() imagesregion = self.get_images_region(regionstr) path = 'syncstatus_' + regionobj.fullname + '.csv' try: tuples = regionobj.image_list_to_sync(self.master_region_dict, imagesregion) tuples.sort(key=lambda tuple: int(tuple[1].size)) writer = csv.writer(stream) for tuple in tuples: (status, image) = tuple l = list() l.append(status) l.append(regionobj.fullname) l.append(image.name) writer.writerow(l) except Exception, e: msg = '{0}: Error retrieving images from region. Cause {1}' msg = msg.format(regionstr, str(e)) self.log.error(msg) raise Exception(msg)
def __init__(self, config_stream=None, options_dict=None): """Constructor of the object """ self.log = logger_cli if config_stream is None: glancesyncconfig = GlanceSyncConfig(override_d=options_dict) else: glancesyncconfig = GlanceSyncConfig(stream=config_stream, override_d=options_dict) self.regions_uris = dict() self.master_region = glancesyncconfig.master_region self.images_dir = glancesyncconfig.images_dir self.targets = glancesyncconfig.targets count = 0 for target in self.targets.values(): if 'GLANCESYNC_USE_MOCK' in os.environ: target['facade'] = ServersFacadeMock(target) elif 'GLANCESYNC_MOCKPERSISTENT_PATH' in os.environ: target['facade'] = ServersFacadeMock(target) target['facade'].init_persistence( os.environ['GLANCESYNC_MOCKPERSISTENT_PATH']) else: target['facade'] = ServersFacade(target) self.preferable_order = glancesyncconfig.preferable_order self.max_children = glancesyncconfig.max_children master_region = GlanceSyncRegion(self.master_region, self.targets) images = master_region.target['facade'].get_imagelist(master_region) self.master_region_dict = self._master_images_to_dict(images) glancesync_ami.clean_ami_ids(self.master_region_dict)
def backup_glancemetadata_region(self, regionstr, path=None): """generate a backup of the metadata on the regional glance server Of course, only data from the tenant and public images are saved!! :param regionstr: The region whose metadata is preserved in a backup :param path: Directory when the file is created (the file it is created in current directory by default) :return: Nothing """ regionobj = GlanceSyncRegion(regionstr, self.targets) if path is None: path = 'backup_' + regionobj.fullname + '.csv' else: path = os.path.join(path, 'backup_' + regionobj.fullname + '.csv') # Backup using csv try: images = regionobj.target['facade'].get_imagelist(regionobj) with open(path, 'w') as csvfile: writer = csv.writer(csvfile) for image in images: writer.writerow(image.to_field_list()) except Exception, e: msg = '{0}:Error retrieving images from region. Cause {1}' msg = msg.format(regionstr, str(e)) self.log.error(msg) raise Exception(msg)
def get_images_region(self, regionstr, only_tenant_images=False): """It returns a list with all the tenant's images in that region :param regionstr: A region specified as 'target:region'. The prefix 'master:' may be omitted. :param only_tenant_images: If true, only include the images owned by the tenant or without owner. :return: a list of GlanceSyncImage objects """ region = GlanceSyncRegion(regionstr, self.targets) facade = region.target['facade'] region.target['tenant_id'] = facade.get_tenant_id() if only_tenant_images: return list( image for image in facade.get_imagelist(region) if not image.owner or image.owner.zfill(32) == region.target['tenant_id'].zfill(32) or image.owner == '') else: return facade.get_imagelist(region)
def get_images_region(self, regionstr, only_tenant_images=False): """It returns a list with all the tenant's images in that region :param regionstr: A region specified as 'target:region'. The prefix 'master:' may be omitted. :param only_tenant_images: If true, only include the images owned by the tenant or without owner. :return: a list of GlanceSyncImage objects """ region = GlanceSyncRegion(regionstr, self.targets) facade = region.target['facade'] region.target['tenant_id'] = facade.get_tenant_id() if only_tenant_images: return list( image for image in facade.get_imagelist(region) if image.name and ( not image.owner or image.owner.zfill(32) == region.target['tenant_id'].zfill(32) or image.owner == '')) else: return facade.get_imagelist(region)
def delete_image(self, regionstr, uuid, confirm=True): """delete a image on the specified region. Be careful, this action cannot be reverted and for this reason by default requires confirmation! :param regionstr: A region specified as 'target:region'. The prefix 'master:' may be omitted. :param uuid: the UUID of the image to delete :param confirm: True to ask for confirmation :return: true if image is deleted, false if canceled """ regionobj = GlanceSyncRegion(regionstr, self.targets) facade = regionobj.target['facade'] return facade.delete_image(regionobj, uuid, confirm)
def update_metadata_image(self, regionstr, image): """update the metadata of the image in the specified region This method takes all the metadata information included in the image and overrides the values of the image with the same name in the region. Important: the synchronisation algorithm filters the metadata using metadata_set, but this method DOES NOT. :param regionstr: A region specified as 'target:region'. The prefix 'master:' may be omitted. :param image: GlanceSyncImage to update on the regional glance server. :return: Nothing """ regionobj = GlanceSyncRegion(regionstr, self.targets) facade = regionobj.target['facade'] facade.update_metadata(regionobj, image)
def sync_region(self, regionstr, dry_run=False): """sync the specified region with the master region Only the images that check the configured condition are synchronised. *If the image is not present on the remote region, is copied from the master region, including the metadata subset specified in metadata_set *If the image is present, but has different properties included in metadata_set, these values are updated, the others are untouched. *If the image has kernel_id and ramdisk_id, it is checked if the ids are from this region. Otherwise, it they are from the master region, they are updated with the images with the same name on this region. It's possible that the image is present in the region, but with different content. This situation is detected comparing the checksums. No image content is overrided, unless specified in configuration. :param regionstr: A region specified as 'target:region'. The prefix 'master:' may be omitted. :param dry_run: If true, images are not uploaded nor modified :return: Nothing """ regionobj = GlanceSyncRegion(regionstr, self.targets) facade = regionobj.target['facade'] target = regionobj.target only_tenant_images = target['only_tenant_images'] target['tenant_id'] = target['facade'].get_tenant_id() imagesregion = self.get_images_region(regionstr, only_tenant_images) # Get a list of obsolete images in the region # they are managed differently that the other images to sync, because: # * they are not uploaded if not present # * the name is changed (the _obsolete suffix is added) if target['support_obsolete_images']: syncprops = target.get('obsolete_syncprops', None) obsolete = regionobj.image_list_to_obsolete( self.master_region_dict, imagesregion, syncprops) else: obsolete = list() # previous step: manage obsolete images. Obsolete images are not # synchronisable. for image in obsolete: self.log.info(regionobj.fullname + ': updating obsolete image ' + image.name) facade.update_metadata(regionobj, image) master_images = regionobj.images_to_sync_dict(self.master_region_dict) dictimages = regionobj.local_images_filtered(master_images, imagesregion) imagesregion = dictimages.values() # Important: tuples are sorted by image.size, in ascending order. This # is important because: # with AMI images, kernel/ramdisk must be uploaded before the image # that refers them. They are smaller. tuples = regionobj.image_list_to_sync(master_images, imagesregion) totalmbs = 0 was_synchronised = True # First, update metadata for tuple in tuples: if tuple[0] == 'pending_metadata': was_synchronised = False if dry_run: self.log.info(regionobj.fullname + ': Image pending to update the metadata ' + tuple[1].name) else: self.log.info(regionobj.fullname + ': Updating the metadata of image ' + tuple[1].name) self.__update_meta(tuple[1], dictimages, regionobj) # Then, upload, replace, and rename_n_replace for tuple in tuples: uploaded = False sizeimage = float(tuple[1].size) / 1024 / 1024 if tuple[0] == 'pending_upload': uploaded = True if not dry_run: self.log.info(regionobj.fullname + ': Uploading image ' + tuple[1].name + ' (' + str(sizeimage) + ' MB)') self.__upload_image(tuple[1], dictimages, regionobj) elif tuple[0] == 'pending_replace': uploaded = True region_image = dictimages[tuple[1].name] self.log.info(regionobj.fullname + ': Replacing image ' + tuple[1].name + ' (' + str(sizeimage) + ' MB)') if not dry_run: self.__upload_image(tuple[1], dictimages, regionobj) facade.delete_image(regionobj, region_image.id, confirm=False) elif tuple[0] == 'pending_rename': uploaded = True region_image = dictimages[tuple[1].name] self.log.info( regionobj.fullname + ': Renaming and replacing image ' + tuple[1].name + ' (' + str(sizeimage) + ' MB)') if not dry_run: self.__upload_image(tuple[1], dictimages, regionobj) region_image.name += '.old' region_image.is_public = False facade.update_metadata(regionobj, region_image) elif tuple[0] == 'error_checksum': region_image = dictimages[tuple[1].name] msg =\ 'Image {0} has a different checksum ({2}) in region {1} '\ 'than in the master region. It was not set what to do. '\ 'Please, fill either dontupdate, replace or rename '\ 'with the checksum.' self.log.warning(msg.format(region_image.name, regionobj.fullname, region_image.checksum)) if uploaded: was_synchronised = False totalmbs += sizeimage if dry_run: self.log.info(regionobj.fullname + ': Pending: ' + tuple[1].name + ' (' + str(sizeimage) + ' MB)') else: self.log.info(regionobj.fullname + ': Image uploaded.') # Finally, update pending AMI ids for tuple in tuples: if tuple[0] == 'pending_ami': self.__update_meta(tuple[1], dictimages, regionobj) if was_synchronised: self.log.info(regionobj.fullname + ': Region is synchronized.') else: if dry_run: self.log.info(regionobj.fullname + ': MBs pending : ' + str(int(totalmbs))) else: self.log.info(regionobj.fullname + ': Total uploaded to region: ' + str(int(totalmbs)) + ' (MB) ')
def sync_region(self, regionstr, dry_run=False): """sync the specified region with the master region Only the images that check the configured condition are synchronised. *If the image is not present on the remote region, is copied from the master region, including the metadata subset specified in metadata_set *If the image is present, but has different properties included in metadata_set, these values are updated, the others are untouched. *If the image has kernel_id and ramdisk_id, it is checked if the ids are from this region. Otherwise, it they are from the master region, they are updated with the images with the same name on this region. It's possible that the image is present in the region, but with different content. This situation is detected comparing the checksums. No image content is overrided, unless specified in configuration. :param regionstr: A region specified as 'target:region'. The prefix 'master:' may be omitted. :param dry_run: If true, images are not uploaded nor modified :return: Nothing """ regionobj = GlanceSyncRegion(regionstr, self.targets) facade = regionobj.target['facade'] target = regionobj.target only_tenant_images = target['only_tenant_images'] target['tenant_id'] = target['facade'].get_tenant_id() imagesregion = self.get_images_region(regionstr, only_tenant_images) # Get a list of obsolete images in the region # they are managed differently that the other images to sync, because: # * they are not uploaded if not present # * the name is changed (the _obsolete suffix is added) if target['support_obsolete_images']: syncprops = target.get('obsolete_syncprops', None) obsolete = regionobj.image_list_to_obsolete( self.master_region_dict, imagesregion, syncprops) else: obsolete = list() # previous step: manage obsolete images. Obsolete images are not # synchronisable. for image in obsolete: self.log.info(regionobj.fullname + ': updating obsolete image ' + image.name) facade.update_metadata(regionobj, image) master_images = regionobj.images_to_sync_dict(self.master_region_dict) dictimages = regionobj.local_images_filtered(master_images, imagesregion) imagesregion = dictimages.values() # Important: tuples are sorted by image.size, in ascending order. This # is important because: # with AMI images, kernel/ramdisk must be uploaded before the image # that refers them. They are smaller. tuples = regionobj.image_list_to_sync(master_images, imagesregion) totalmbs = 0 was_synchronised = True # First, update metadata for tuple in tuples: if tuple[0] == 'pending_metadata': was_synchronised = False if dry_run: self.log.info(regionobj.fullname + ': Image pending to update the metadata ' + tuple[1].name) else: self.log.info(regionobj.fullname + ': Updating the metadata of image ' + tuple[1].name) self.__update_meta(tuple[1], dictimages, regionobj) # Then, upload, replace, and rename_n_replace for tuple in tuples: uploaded = False sizeimage = float(tuple[1].size) / 1024 / 1024 if tuple[0] == 'pending_upload': uploaded = True if not dry_run: self.log.info(regionobj.fullname + ': Uploading image ' + tuple[1].name + ' (' + str(sizeimage) + ' MB)') self.__upload_image(tuple[1], dictimages, regionobj) elif tuple[0] == 'pending_replace': uploaded = True region_image = dictimages[tuple[1].name] self.log.info(regionobj.fullname + ': Replacing image ' + tuple[1].name + ' (' + str(sizeimage) + ' MB)') if not dry_run: self.__upload_image(tuple[1], dictimages, regionobj) facade.delete_image(regionobj, region_image.id, confirm=False) elif tuple[0] == 'pending_rename': uploaded = True region_image = dictimages[tuple[1].name] self.log.info(regionobj.fullname + ': Renaming and replacing image ' + tuple[1].name + ' (' + str(sizeimage) + ' MB)') if not dry_run: self.__upload_image(tuple[1], dictimages, regionobj) region_image.name += '.old' region_image.is_public = False facade.update_metadata(regionobj, region_image) elif tuple[0] == 'error_checksum': region_image = dictimages[tuple[1].name] msg =\ 'Image {0} has a different checksum ({2}) in region {1} '\ 'than in the master region. It was not set what to do. '\ 'Please, fill either dontupdate, replace or rename '\ 'with the checksum.' self.log.warning( msg.format(region_image.name, regionobj.fullname, region_image.checksum)) if uploaded: was_synchronised = False totalmbs += sizeimage if dry_run: self.log.info(regionobj.fullname + ': Pending: ' + tuple[1].name + ' (' + str(sizeimage) + ' MB)') else: self.log.info(regionobj.fullname + ': Image uploaded.') # Finally, update pending AMI ids for tuple in tuples: if tuple[0] == 'pending_ami': self.__update_meta(tuple[1], dictimages, regionobj) if was_synchronised: self.log.info(regionobj.fullname + ': Region is synchronized.') else: if dry_run: self.log.info(regionobj.fullname + ': MBs pending : ' + str(int(totalmbs))) else: self.log.info(regionobj.fullname + ': Total uploaded to region: ' + str(int(totalmbs)) + ' (MB) ')