def __get_or_create_manifest(self, vol_container, file_size): _, bucket, key = self.args['s3_service'].resolve_url_to_location( vol_container['image']['importManifestUrl']) manifest_s3path = '/'.join((bucket, key)) try: with tempfile.SpooledTemporaryFile(max_size=1024000) as \ manifest_destfile: get_req = GetObject.from_other( self, service=self.args['s3_service'], auth=self.args['s3_auth'], source=manifest_s3path, dest=manifest_destfile, show_progress=False) get_req.main() self.log.info('using existing import manifest from the server') manifest_destfile.seek(0) manifest = ImportManifest.read_from_fileobj( manifest_destfile) except ServerError as err: if err.status_code == 404: self.log.info('creating new import manifest') manifest = self.__generate_manifest(vol_container, file_size) tempdir = tempfile.mkdtemp() manifest_filename = os.path.join(tempdir, os.path.basename(key)) with open(manifest_filename, 'w') as manifest_file: manifest.dump_to_fileobj(manifest_file, pretty_print=True) put_req = PutObject.from_other( get_req, source=manifest_filename, dest=manifest_s3path, show_progress=False) put_req.main() os.remove(manifest_filename) os.rmdir(tempdir) else: raise return manifest
def __generate_manifest(self, vol_container, file_size): days = self.args.get('expires') or 30 timeout = days * 86400 # in seconds _, bucket, key = self.args['s3_service'].resolve_url_to_location( vol_container['image']['importManifestUrl']) key_prefix = key.rsplit('/', 1)[0] manifest = ImportManifest(loglevel=self.log.level) manifest.file_format = vol_container['image']['format'] delete_req = DeleteObject.from_other( self, service=self.args['s3_service'], auth=self.args['s3_auth'], path='/'.join((bucket, key))) manifest.self_destruct_url = delete_req.get_presigned_url2(timeout) manifest.image_size = int(vol_container['image']['size']) manifest.volume_size = int(vol_container['volume']['size']) part_size = (self.args.get('part_size') or 10) * 2 ** 20 # MiB for index, part_start in enumerate(six.moves.range(0, file_size, part_size)): part = ImportImagePart() part.index = index part.start = part_start part.end = min(part_start + part_size, file_size) - 1 part.key = '{0}/{1}.part.{2}'.format( key_prefix, os.path.basename(self.args['source']), index) part_path = '/'.join((bucket, part.key)) head_req = HeadObject.from_other(delete_req, path=part_path) get_req = GetObject.from_other(delete_req, source=part_path) delete_req = DeleteObject.from_other(delete_req, path=part_path) part.head_url = head_req.get_presigned_url2(timeout) part.get_url = get_req.get_presigned_url2(timeout) part.delete_url = delete_req.get_presigned_url2(timeout) manifest.image_parts.append(part) return manifest
def __download_manifest(self, s3path): with tempfile.SpooledTemporaryFile(max_size=1024000) as \ manifest_destfile: get_req = GetObject.from_other(self, service=self.args['s3_service'], auth=self.args['s3_auth'], source=s3path, dest=manifest_destfile, show_progress=False) try: get_req.main() except AWSError as err: if err.status_code == 404: raise ArgumentError( 'import manifest "{0}" does not exist'.format(s3path)) raise manifest_destfile.seek(0) return ImportManifest.read_from_fileobj(manifest_destfile)
def __download_manifest(self, s3path): with tempfile.SpooledTemporaryFile(max_size=1024000) as manifest_destfile: get_req = GetObject.from_other( self, service=self.args["s3_service"], auth=self.args["s3_auth"], source=s3path, dest=manifest_destfile, show_progress=False, ) try: get_req.main() except AWSError as err: if err.status_code == 404: raise ArgumentError('import manifest "{0}" does not exist'.format(s3path)) raise manifest_destfile.seek(0) return ImportManifest.read_from_fileobj(manifest_destfile)