def __generate_manifest(self, vol_container, file_size): days = self.args.get('expires') or 30 timeout = days * 86400 # in seconds _, bucket, key = self.args['s3_service'].resolve_url_to_location( vol_container['image']['importManifestUrl']) key_prefix = key.rsplit('/', 1)[0] manifest = ImportManifest(loglevel=self.log.level) manifest.file_format = vol_container['image']['format'] delete_req = DeleteObject.from_other( self, service=self.args['s3_service'], auth=self.args['s3_auth'], path='/'.join((bucket, key))) manifest.self_destruct_url = delete_req.get_presigned_url2(timeout) manifest.image_size = int(vol_container['image']['size']) manifest.volume_size = int(vol_container['volume']['size']) part_size = (self.args.get('part_size') or 10) * 2 ** 20 # MiB for index, part_start in enumerate(six.moves.range(0, file_size, part_size)): part = ImportImagePart() part.index = index part.start = part_start part.end = min(part_start + part_size, file_size) - 1 part.key = '{0}/{1}.part.{2}'.format( key_prefix, os.path.basename(self.args['source']), index) part_path = '/'.join((bucket, part.key)) head_req = HeadObject.from_other(delete_req, path=part_path) get_req = GetObject.from_other(delete_req, source=part_path) delete_req = DeleteObject.from_other(delete_req, path=part_path) part.head_url = head_req.get_presigned_url2(timeout) part.get_url = get_req.get_presigned_url2(timeout) part.delete_url = delete_req.get_presigned_url2(timeout) manifest.image_parts.append(part) return manifest
def main(self): try: manifest = self.fetch_manifest(self.service) except ServerError as err: if err.status_code == 404 and self.args.get('clear'): try: # We are supposed to try to delete the bucket even # if the manifest isn't there. If it works, the # bundle is also gone and we can safely return. # # https://eucalyptus.atlassian.net/browse/TOOLS-379 self.__delete_bucket() return except ServerError: # If the bucket wasn't empty then we'll go back to # complaining about the missing manifest. self.log.error( 'failed to delete bucket %s after a failed ' 'attempt to fetch the bundle manifest', bucket=self.args['bucket'].split('/')[0], exc_info=True) raise for _, part_s3path in self.map_bundle_parts_to_s3paths(manifest): req = DeleteObject.from_other(self, path=part_s3path) req.main() manifest_s3path = self.get_manifest_s3path() if manifest_s3path: req = DeleteObject.from_other(self, path=manifest_s3path) req.main() if self.args.get('clear'): self.__delete_bucket()
def main(self): if self.args.get('manifest_url'): manifest_url = self.args['manifest_url'] if self.args.get('task'): desc_conv = DescribeConversionTasks.from_other( self, ConversionTaskId=[self.args['task']]) task = desc_conv.main()['conversionTasks'][0] assert task['conversionTaskId'] == self.args['task'] if task.get('importVolume'): vol_container = task['importVolume'] else: vol_container = task['importInstance']['volumes'][0] manifest_url = vol_container['image']['importManifestUrl'] _, bucket, key = self.args['s3_service'].resolve_url_to_location( manifest_url) manifest_s3path = '/'.join((bucket, key)) manifest = self.__download_manifest(manifest_s3path) for part in manifest.image_parts: delete_req = DeleteObject.from_other( self, service=self.args['s3_service'], auth=self.args['s3_auth'], path='/'.join((bucket, part.key))) delete_req.main() delete_req = DeleteObject.from_other(self, service=self.args['s3_service'], auth=self.args['s3_auth'], path=manifest_s3path) delete_req.main()
def main(self): if self.args.get("manifest_url"): manifest_url = self.args["manifest_url"] if self.args.get("task"): desc_conv = DescribeConversionTasks.from_other(self, ConversionTaskId=[self.args["task"]]) task = desc_conv.main()["conversionTasks"][0] assert task["conversionTaskId"] == self.args["task"] if task.get("importVolume"): vol_container = task["importVolume"] else: vol_container = task["importInstance"]["volumes"][0] manifest_url = vol_container["image"]["importManifestUrl"] _, bucket, key = self.args["s3_service"].resolve_url_to_location(manifest_url) manifest_s3path = "/".join((bucket, key)) manifest = self.__download_manifest(manifest_s3path) for part in manifest.image_parts: delete_req = DeleteObject.from_other( self, service=self.args["s3_service"], auth=self.args["s3_auth"], path="/".join((bucket, part.key)) ) delete_req.main() delete_req = DeleteObject.from_other( self, service=self.args["s3_service"], auth=self.args["s3_auth"], path=manifest_s3path ) delete_req.main()