def __generate_manifest(self, vol_container, file_size):
days = self.args.get('expires') or 30
timeout = days * 86400 # in seconds
_, bucket, key = self.args['s3_service'].resolve_url_to_location(
vol_container['image']['importManifestUrl'])
key_prefix = key.rsplit('/', 1)[0]
manifest = ImportManifest(loglevel=self.log.level)
manifest.file_format = vol_container['image']['format']
delete_req = DeleteObject.from_other(
self, service=self.args['s3_service'], auth=self.args['s3_auth'],
path='/'.join((bucket, key)))
manifest.self_destruct_url = delete_req.get_presigned_url2(timeout)
manifest.image_size = int(vol_container['image']['size'])
manifest.volume_size = int(vol_container['volume']['size'])
part_size = (self.args.get('part_size') or 10) * 2 ** 20 # MiB
for index, part_start in enumerate(six.moves.range(0, file_size,
part_size)):
part = ImportImagePart()
part.index = index
part.start = part_start
part.end = min(part_start + part_size, file_size) - 1
part.key = '{0}/{1}.part.{2}'.format(
key_prefix, os.path.basename(self.args['source']), index)
part_path = '/'.join((bucket, part.key))
head_req = HeadObject.from_other(delete_req, path=part_path)
get_req = GetObject.from_other(delete_req, source=part_path)
delete_req = DeleteObject.from_other(delete_req, path=part_path)
part.head_url = head_req.get_presigned_url2(timeout)
part.get_url = get_req.get_presigned_url2(timeout)
part.delete_url = delete_req.get_presigned_url2(timeout)
manifest.image_parts.append(part)
return manifest
def main(self):
try:
manifest = self.fetch_manifest(self.service)
except ServerError as err:
if err.status_code == 404 and self.args.get('clear'):
try:
# We are supposed to try to delete the bucket even
# if the manifest isn't there. If it works, the
# bundle is also gone and we can safely return.
#
# https://eucalyptus.atlassian.net/browse/TOOLS-379
self.__delete_bucket()
return
except ServerError:
# If the bucket wasn't empty then we'll go back to
# complaining about the missing manifest.
self.log.error(
'failed to delete bucket %s after a failed '
'attempt to fetch the bundle manifest',
bucket=self.args['bucket'].split('/')[0],
exc_info=True)
raise
for _, part_s3path in self.map_bundle_parts_to_s3paths(manifest):
req = DeleteObject.from_other(self, path=part_s3path)
req.main()
manifest_s3path = self.get_manifest_s3path()
if manifest_s3path:
req = DeleteObject.from_other(self, path=manifest_s3path)
req.main()
if self.args.get('clear'):
self.__delete_bucket()
def main(self):
if self.args.get('manifest_url'):
manifest_url = self.args['manifest_url']
if self.args.get('task'):
desc_conv = DescribeConversionTasks.from_other(
self, ConversionTaskId=[self.args['task']])
task = desc_conv.main()['conversionTasks'][0]
assert task['conversionTaskId'] == self.args['task']
if task.get('importVolume'):
vol_container = task['importVolume']
else:
vol_container = task['importInstance']['volumes'][0]
manifest_url = vol_container['image']['importManifestUrl']
_, bucket, key = self.args['s3_service'].resolve_url_to_location(
manifest_url)
manifest_s3path = '/'.join((bucket, key))
manifest = self.__download_manifest(manifest_s3path)
for part in manifest.image_parts:
delete_req = DeleteObject.from_other(
self,
service=self.args['s3_service'],
auth=self.args['s3_auth'],
path='/'.join((bucket, part.key)))
delete_req.main()
delete_req = DeleteObject.from_other(self,
service=self.args['s3_service'],
auth=self.args['s3_auth'],
path=manifest_s3path)
delete_req.main()
def main(self):
if self.args.get("manifest_url"):
manifest_url = self.args["manifest_url"]
if self.args.get("task"):
desc_conv = DescribeConversionTasks.from_other(self, ConversionTaskId=[self.args["task"]])
task = desc_conv.main()["conversionTasks"][0]
assert task["conversionTaskId"] == self.args["task"]
if task.get("importVolume"):
vol_container = task["importVolume"]
else:
vol_container = task["importInstance"]["volumes"][0]
manifest_url = vol_container["image"]["importManifestUrl"]
_, bucket, key = self.args["s3_service"].resolve_url_to_location(manifest_url)
manifest_s3path = "/".join((bucket, key))
manifest = self.__download_manifest(manifest_s3path)
for part in manifest.image_parts:
delete_req = DeleteObject.from_other(
self, service=self.args["s3_service"], auth=self.args["s3_auth"], path="/".join((bucket, part.key))
)
delete_req.main()
delete_req = DeleteObject.from_other(
self, service=self.args["s3_service"], auth=self.args["s3_auth"], path=manifest_s3path
)
delete_req.main()
