def fetch_manifest(self, s3_service, privkey_filename=None):
if self.args.get('local_manifest'):
_assert_is_file(self.args['local_manifest'], 'manifest')
return euca2ools.bundle.manifest.BundleManifest.read_from_file(
self.args['local_manifest'], privkey_filename=privkey_filename)
# It's on the server, so do things the hard way
manifest_s3path = self.get_manifest_s3path()
with tempfile.TemporaryFile() as manifest_tempfile:
self.log.info('reading manifest from %s', manifest_s3path)
req = GetObject(config=self.config,
service=s3_service,
source=manifest_s3path,
dest=manifest_tempfile)
try:
req.main()
except AWSError as err:
if err.status_code == 404:
self.log.debug('failed to fetch manifest', exc_info=True)
raise ValueError("manifest '{0}' does not exist on the "
"server".format(manifest_s3path))
raise
manifest_tempfile.flush()
manifest_tempfile.seek(0)
return euca2ools.bundle.manifest.BundleManifest.read_from_fileobj(
manifest_tempfile, privkey_filename=privkey_filename)
def download_bundle_to_dir(self, manifest, dest_dir, s3_service):
parts = self.map_bundle_parts_to_s3paths(manifest)
for part, part_s3path in parts:
part.filename = os.path.join(dest_dir,
os.path.basename(part_s3path))
self.log.info('downloading part %s to %s',
part_s3path, part.filename)
req = GetObject.from_other(
self, service=s3_service, source=part_s3path,
dest=part.filename,
show_progress=self.args.get('show_progress', False))
response = req.main()
self.__check_part_sha1(part, part_s3path, response)
manifest_s3path = self.get_manifest_s3path()
if manifest_s3path:
# Can't download a manifest if we're using a local one
manifest_dest = os.path.join(dest_dir,
os.path.basename(manifest_s3path))
self.log.info('downloading manifest %s to %s',
manifest_s3path, manifest_dest)
req = GetObject.from_other(
self, service=s3_service, source=manifest_s3path,
dest=manifest_dest,
show_progress=self.args.get('show_progress', False))
req.main()
return manifest_dest
return None
def download_bundle_to_dir(self, manifest, dest_dir, s3_service):
parts = self.map_bundle_parts_to_s3paths(manifest)
for part, part_s3path in parts:
part.filename = os.path.join(dest_dir,
os.path.basename(part_s3path))
self.log.info('downloading part %s to %s', part_s3path,
part.filename)
req = GetObject.from_other(self,
service=s3_service,
source=part_s3path,
dest=part.filename,
show_progress=self.args.get(
'show_progress', False))
response = req.main()
self.__check_part_sha1(part, part_s3path, response)
manifest_s3path = self.get_manifest_s3path()
if manifest_s3path:
# Can't download a manifest if we're using a local one
manifest_dest = os.path.join(dest_dir,
os.path.basename(manifest_s3path))
self.log.info('downloading manifest %s to %s', manifest_s3path,
manifest_dest)
req = GetObject.from_other(self,
service=s3_service,
source=manifest_s3path,
dest=manifest_dest,
show_progress=self.args.get(
'show_progress', False))
req.main()
return manifest_dest
return None
def download_bundle_to_fileobj(self, manifest, fileobj, s3_service):
# We can skip downloading the manifest since we're just writing all
# parts to a file object.
parts = self.map_bundle_parts_to_s3paths(manifest)
for part, part_s3path in parts:
self.log.info('downloading part %s', part_s3path)
req = GetObject(
config=self.config, service=s3_service,
source=part_s3path, dest=fileobj,
show_progress=self.args.get('show_progress', False))
response = req.main()
self.__check_part_sha1(part, part_s3path, response)
def download_bundle_to_fileobj(self, manifest, fileobj, s3_service):
# We can skip downloading the manifest since we're just writing all
# parts to a file object.
parts = self.map_bundle_parts_to_s3paths(manifest)
for part, part_s3path in parts:
self.log.info('downloading part %s', part_s3path)
req = GetObject(config=self.config,
service=s3_service,
source=part_s3path,
dest=fileobj,
show_progress=self.args.get(
'show_progress', False))
response = req.main()
self.__check_part_sha1(part, part_s3path, response)
def fetch_manifest(self, s3_service, privkey_filename=None):
if self.args.get('local_manifest'):
_assert_is_file(self.args['local_manifest'], 'manifest')
return euca2ools.bundle.manifest.BundleManifest.read_from_file(
self.args['local_manifest'], privkey_filename=privkey_filename)
# It's on the server, so do things the hard way
manifest_s3path = self.get_manifest_s3path()
with tempfile.TemporaryFile() as manifest_tempfile:
self.log.info('reading manifest from %s', manifest_s3path)
req = GetObject.from_other(
self, service=s3_service, source=manifest_s3path,
dest=manifest_tempfile)
try:
req.main()
except AWSError as err:
if err.status_code == 404:
self.log.debug('failed to fetch manifest', exc_info=True)
raise ValueError("manifest '{0}' does not exist on the "
"server".format(manifest_s3path))
raise
manifest_tempfile.flush()
manifest_tempfile.seek(0)
return euca2ools.bundle.manifest.BundleManifest.read_from_fileobj(
manifest_tempfile, privkey_filename=privkey_filename)
def main(self):
if self.args.get('dry_run'):
return
if not self.args.get('DiskImage.1.Image.ImportManifestUrl'):
manifest_key = '{0}/{1}.manifest.xml'.format(uuid.uuid4(),
self.args['source'])
if self.args.get('prefix'):
manifest_key = '/'.join((self.args['prefix'], manifest_key))
getobj = GetObject.from_other(
self, service=self.args['s3_service'],
auth=self.args['s3_auth'],
source='/'.join((self.args['bucket'], manifest_key)))
days = self.args.get('expires') or 30
expiration = datetime.datetime.utcnow() + datetime.timedelta(days)
get_url = getobj.get_presigned_url(expiration)
self.log.info('generated manifest GET URL: %s', get_url)
self.params['DiskImage.1.Image.ImportManifestUrl'] = get_url
result = self.send()
# The manifest creation and uploading parts are done by ResumeImport.
if not self.args.get('no_upload'):
resume = ResumeImport.from_other(
self, source=self.args['source'],
task=result['conversionTask']['conversionTaskId'],
s3_service=self.args['s3_service'],
s3_auth=self.args['s3_auth'], expires=self.args['expires'],
show_progress=self.args.get('show_progress', False))
resume.main()
return result
def __generate_manifest(self, vol_container, file_size):
days = self.args.get('expires') or 30
timeout = days * 86400 # in seconds
_, bucket, key = self.args['s3_service'].resolve_url_to_location(
vol_container['image']['importManifestUrl'])
key_prefix = key.rsplit('/', 1)[0]
manifest = ImportManifest(loglevel=self.log.level)
manifest.file_format = vol_container['image']['format']
delete_req = DeleteObject.from_other(
self, service=self.args['s3_service'], auth=self.args['s3_auth'],
path='/'.join((bucket, key)))
manifest.self_destruct_url = delete_req.get_presigned_url2(timeout)
manifest.image_size = int(vol_container['image']['size'])
manifest.volume_size = int(vol_container['volume']['size'])
part_size = (self.args.get('part_size') or 10) * 2 ** 20 # MiB
for index, part_start in enumerate(six.moves.range(0, file_size,
part_size)):
part = ImportImagePart()
part.index = index
part.start = part_start
part.end = min(part_start + part_size, file_size) - 1
part.key = '{0}/{1}.part.{2}'.format(
key_prefix, os.path.basename(self.args['source']), index)
part_path = '/'.join((bucket, part.key))
head_req = HeadObject.from_other(delete_req, path=part_path)
get_req = GetObject.from_other(delete_req, source=part_path)
delete_req = DeleteObject.from_other(delete_req, path=part_path)
part.head_url = head_req.get_presigned_url2(timeout)
part.get_url = get_req.get_presigned_url2(timeout)
part.delete_url = delete_req.get_presigned_url2(timeout)
manifest.image_parts.append(part)
return manifest
def __get_or_create_manifest(self, vol_container, file_size):
_, bucket, key = self.args['s3_service'].resolve_url_to_location(
vol_container['image']['importManifestUrl'])
manifest_s3path = '/'.join((bucket, key))
try:
with tempfile.SpooledTemporaryFile(max_size=1024000) as \
manifest_destfile:
get_req = GetObject.from_other(
self, service=self.args['s3_service'],
auth=self.args['s3_auth'], source=manifest_s3path,
dest=manifest_destfile, show_progress=False)
get_req.main()
self.log.info('using existing import manifest from the server')
manifest_destfile.seek(0)
manifest = ImportManifest.read_from_fileobj(
manifest_destfile)
except ServerError as err:
if err.status_code == 404:
self.log.info('creating new import manifest')
manifest = self.__generate_manifest(vol_container, file_size)
tempdir = tempfile.mkdtemp()
manifest_filename = os.path.join(tempdir,
os.path.basename(key))
with open(manifest_filename, 'w') as manifest_file:
manifest.dump_to_fileobj(manifest_file, pretty_print=True)
put_req = PutObject.from_other(
get_req, source=manifest_filename, dest=manifest_s3path,
show_progress=False)
put_req.main()
os.remove(manifest_filename)
os.rmdir(tempdir)
else:
raise
return manifest
def main(self):
if self.args.get('dry_run'):
return
if not self.args.get('Image.ImportManifestUrl'):
manifest_key = '{0}/{1}.manifest.xml'.format(
uuid.uuid4(), self.args['source'])
if self.args.get('prefix'):
manifest_key = '/'.join((self.args['prefix'], manifest_key))
getobj = GetObject.from_other(self,
service=self.args['s3_service'],
auth=self.args['s3_auth'],
source='/'.join((self.args['bucket'],
manifest_key)))
days = self.args.get('expires') or 30
expiration = datetime.datetime.utcnow() + datetime.timedelta(days)
get_url = getobj.get_presigned_url(expiration)
self.log.info('generated manifest GET URL: %s', get_url)
self.params['Image.ImportManifestUrl'] = get_url
result = self.send()
# The manifest creation and uploading parts are done by ResumeImport.
if not self.args.get('no_upload'):
resume = ResumeImport.from_other(
self,
source=self.args['source'],
task=result['conversionTask']['conversionTaskId'],
s3_service=self.args['s3_service'],
s3_auth=self.args['s3_auth'],
expires=self.args['expires'],
show_progress=self.args.get('show_progress', False))
resume.main()
return result
def __download_manifest(self, s3path):
with tempfile.SpooledTemporaryFile(max_size=1024000) as \
manifest_destfile:
get_req = GetObject.from_other(self,
service=self.args['s3_service'],
auth=self.args['s3_auth'],
source=s3path,
dest=manifest_destfile,
show_progress=False)
try:
get_req.main()
except AWSError as err:
if err.status_code == 404:
raise ArgumentError(
'import manifest "{0}" does not exist'.format(s3path))
raise
manifest_destfile.seek(0)
return ImportManifest.read_from_fileobj(manifest_destfile)
def __download_manifest(self, s3path):
with tempfile.SpooledTemporaryFile(max_size=1024000) as manifest_destfile:
get_req = GetObject.from_other(
self,
service=self.args["s3_service"],
auth=self.args["s3_auth"],
source=s3path,
dest=manifest_destfile,
show_progress=False,
)
try:
get_req.main()
except AWSError as err:
if err.status_code == 404:
raise ArgumentError('import manifest "{0}" does not exist'.format(s3path))
raise
manifest_destfile.seek(0)
return ImportManifest.read_from_fileobj(manifest_destfile)
def main(self):
if self.args.get("dry_run"):
return
if self.args.get("bucket"):
self.ensure_bucket_exists(self.args["bucket"])
if not self.args.get("DiskImage.1.Image.ImportManifestUrl"):
manifest_key = "{0}/{1}.manifest.xml".format(uuid.uuid4(), self.args["source"])
if self.args.get("prefix"):
manifest_key = "/".join((self.args["prefix"], manifest_key))
getobj = GetObject.from_other(
self,
service=self.args["s3_service"],
auth=self.args["s3_auth"],
source="/".join((self.args["bucket"], manifest_key)),
)
days = self.args.get("expires") or 30
expiration = datetime.datetime.utcnow() + datetime.timedelta(days)
get_url = getobj.get_presigned_url(expiration)
self.log.info("generated manifest GET URL: %s", get_url)
self.params["DiskImage.1.Image.ImportManifestUrl"] = get_url
result = self.send()
# The manifest creation and uploading parts are done by ResumeImport.
if not self.args.get("no_upload"):
resume = ResumeImport.from_other(
self,
source=self.args["source"],
task=result["conversionTask"]["conversionTaskId"],
s3_service=self.args["s3_service"],
s3_auth=self.args["s3_auth"],
expires=self.args["expires"],
show_progress=self.args.get("show_progress", False),
)
resume.main()
return result
