def __get_or_create_manifest(self, vol_container, file_size):
_, bucket, key = self.args['s3_service'].resolve_url_to_location(
vol_container['image']['importManifestUrl'])
manifest_s3path = '/'.join((bucket, key))
try:
with tempfile.SpooledTemporaryFile(max_size=1024000) as \
manifest_destfile:
get_req = GetObject.from_other(
self, service=self.args['s3_service'],
auth=self.args['s3_auth'], source=manifest_s3path,
dest=manifest_destfile, show_progress=False)
get_req.main()
self.log.info('using existing import manifest from the server')
manifest_destfile.seek(0)
manifest = ImportManifest.read_from_fileobj(
manifest_destfile)
except ServerError as err:
if err.status_code == 404:
self.log.info('creating new import manifest')
manifest = self.__generate_manifest(vol_container, file_size)
tempdir = tempfile.mkdtemp()
manifest_filename = os.path.join(tempdir,
os.path.basename(key))
with open(manifest_filename, 'w') as manifest_file:
manifest.dump_to_fileobj(manifest_file, pretty_print=True)
put_req = PutObject.from_other(
get_req, source=manifest_filename, dest=manifest_s3path,
show_progress=False)
put_req.main()
os.remove(manifest_filename)
os.rmdir(tempdir)
else:
raise
return manifest
def __generate_manifest(self, vol_container, file_size):
days = self.args.get('expires') or 30
timeout = days * 86400 # in seconds
_, bucket, key = self.args['s3_service'].resolve_url_to_location(
vol_container['image']['importManifestUrl'])
key_prefix = key.rsplit('/', 1)[0]
manifest = ImportManifest(loglevel=self.log.level)
manifest.file_format = vol_container['image']['format']
delete_req = DeleteObject.from_other(
self, service=self.args['s3_service'], auth=self.args['s3_auth'],
path='/'.join((bucket, key)))
manifest.self_destruct_url = delete_req.get_presigned_url2(timeout)
manifest.image_size = int(vol_container['image']['size'])
manifest.volume_size = int(vol_container['volume']['size'])
part_size = (self.args.get('part_size') or 10) * 2 ** 20 # MiB
for index, part_start in enumerate(six.moves.range(0, file_size,
part_size)):
part = ImportImagePart()
part.index = index
part.start = part_start
part.end = min(part_start + part_size, file_size) - 1
part.key = '{0}/{1}.part.{2}'.format(
key_prefix, os.path.basename(self.args['source']), index)
part_path = '/'.join((bucket, part.key))
head_req = HeadObject.from_other(delete_req, path=part_path)
get_req = GetObject.from_other(delete_req, source=part_path)
delete_req = DeleteObject.from_other(delete_req, path=part_path)
part.head_url = head_req.get_presigned_url2(timeout)
part.get_url = get_req.get_presigned_url2(timeout)
part.delete_url = delete_req.get_presigned_url2(timeout)
manifest.image_parts.append(part)
return manifest
def __download_manifest(self, s3path):
with tempfile.SpooledTemporaryFile(max_size=1024000) as \
manifest_destfile:
get_req = GetObject.from_other(self,
service=self.args['s3_service'],
auth=self.args['s3_auth'],
source=s3path,
dest=manifest_destfile,
show_progress=False)
try:
get_req.main()
except AWSError as err:
if err.status_code == 404:
raise ArgumentError(
'import manifest "{0}" does not exist'.format(s3path))
raise
manifest_destfile.seek(0)
return ImportManifest.read_from_fileobj(manifest_destfile)
def __download_manifest(self, s3path):
with tempfile.SpooledTemporaryFile(max_size=1024000) as manifest_destfile:
get_req = GetObject.from_other(
self,
service=self.args["s3_service"],
auth=self.args["s3_auth"],
source=s3path,
dest=manifest_destfile,
show_progress=False,
)
try:
get_req.main()
except AWSError as err:
if err.status_code == 404:
raise ArgumentError('import manifest "{0}" does not exist'.format(s3path))
raise
manifest_destfile.seek(0)
return ImportManifest.read_from_fileobj(manifest_destfile)
