def download_bundle_to_dir(self, manifest, dest_dir, s3_service): parts = self.map_bundle_parts_to_s3paths(manifest) for part, part_s3path in parts: part.filename = os.path.join(dest_dir, os.path.basename(part_s3path)) self.log.info('downloading part %s to %s', part_s3path, part.filename) req = GetObject.from_other( self, service=s3_service, source=part_s3path, dest=part.filename, show_progress=self.args.get('show_progress', False)) response = req.main() self.__check_part_sha1(part, part_s3path, response) manifest_s3path = self.get_manifest_s3path() if manifest_s3path: # Can't download a manifest if we're using a local one manifest_dest = os.path.join(dest_dir, os.path.basename(manifest_s3path)) self.log.info('downloading manifest %s to %s', manifest_s3path, manifest_dest) req = GetObject.from_other( self, service=s3_service, source=manifest_s3path, dest=manifest_dest, show_progress=self.args.get('show_progress', False)) req.main() return manifest_dest return None
def download_bundle_to_dir(self, manifest, dest_dir, s3_service): parts = self.map_bundle_parts_to_s3paths(manifest) for part, part_s3path in parts: part.filename = os.path.join(dest_dir, os.path.basename(part_s3path)) self.log.info('downloading part %s to %s', part_s3path, part.filename) req = GetObject.from_other(self, service=s3_service, source=part_s3path, dest=part.filename, show_progress=self.args.get( 'show_progress', False)) response = req.main() self.__check_part_sha1(part, part_s3path, response) manifest_s3path = self.get_manifest_s3path() if manifest_s3path: # Can't download a manifest if we're using a local one manifest_dest = os.path.join(dest_dir, os.path.basename(manifest_s3path)) self.log.info('downloading manifest %s to %s', manifest_s3path, manifest_dest) req = GetObject.from_other(self, service=s3_service, source=manifest_s3path, dest=manifest_dest, show_progress=self.args.get( 'show_progress', False)) req.main() return manifest_dest return None
def fetch_manifest(self, s3_service, privkey_filename=None): if self.args.get('local_manifest'): _assert_is_file(self.args['local_manifest'], 'manifest') return euca2ools.bundle.manifest.BundleManifest.read_from_file( self.args['local_manifest'], privkey_filename=privkey_filename) # It's on the server, so do things the hard way manifest_s3path = self.get_manifest_s3path() with tempfile.TemporaryFile() as manifest_tempfile: self.log.info('reading manifest from %s', manifest_s3path) req = GetObject.from_other( self, service=s3_service, source=manifest_s3path, dest=manifest_tempfile) try: req.main() except AWSError as err: if err.status_code == 404: self.log.debug('failed to fetch manifest', exc_info=True) raise ValueError("manifest '{0}' does not exist on the " "server".format(manifest_s3path)) raise manifest_tempfile.flush() manifest_tempfile.seek(0) return euca2ools.bundle.manifest.BundleManifest.read_from_fileobj( manifest_tempfile, privkey_filename=privkey_filename)
def main(self): if self.args.get('dry_run'): return if not self.args.get('DiskImage.1.Image.ImportManifestUrl'): manifest_key = '{0}/{1}.manifest.xml'.format(uuid.uuid4(), self.args['source']) if self.args.get('prefix'): manifest_key = '/'.join((self.args['prefix'], manifest_key)) getobj = GetObject.from_other( self, service=self.args['s3_service'], auth=self.args['s3_auth'], source='/'.join((self.args['bucket'], manifest_key))) days = self.args.get('expires') or 30 expiration = datetime.datetime.utcnow() + datetime.timedelta(days) get_url = getobj.get_presigned_url(expiration) self.log.info('generated manifest GET URL: %s', get_url) self.params['DiskImage.1.Image.ImportManifestUrl'] = get_url result = self.send() # The manifest creation and uploading parts are done by ResumeImport. if not self.args.get('no_upload'): resume = ResumeImport.from_other( self, source=self.args['source'], task=result['conversionTask']['conversionTaskId'], s3_service=self.args['s3_service'], s3_auth=self.args['s3_auth'], expires=self.args['expires'], show_progress=self.args.get('show_progress', False)) resume.main() return result
def __generate_manifest(self, vol_container, file_size): days = self.args.get('expires') or 30 timeout = days * 86400 # in seconds _, bucket, key = self.args['s3_service'].resolve_url_to_location( vol_container['image']['importManifestUrl']) key_prefix = key.rsplit('/', 1)[0] manifest = ImportManifest(loglevel=self.log.level) manifest.file_format = vol_container['image']['format'] delete_req = DeleteObject.from_other( self, service=self.args['s3_service'], auth=self.args['s3_auth'], path='/'.join((bucket, key))) manifest.self_destruct_url = delete_req.get_presigned_url2(timeout) manifest.image_size = int(vol_container['image']['size']) manifest.volume_size = int(vol_container['volume']['size']) part_size = (self.args.get('part_size') or 10) * 2 ** 20 # MiB for index, part_start in enumerate(six.moves.range(0, file_size, part_size)): part = ImportImagePart() part.index = index part.start = part_start part.end = min(part_start + part_size, file_size) - 1 part.key = '{0}/{1}.part.{2}'.format( key_prefix, os.path.basename(self.args['source']), index) part_path = '/'.join((bucket, part.key)) head_req = HeadObject.from_other(delete_req, path=part_path) get_req = GetObject.from_other(delete_req, source=part_path) delete_req = DeleteObject.from_other(delete_req, path=part_path) part.head_url = head_req.get_presigned_url2(timeout) part.get_url = get_req.get_presigned_url2(timeout) part.delete_url = delete_req.get_presigned_url2(timeout) manifest.image_parts.append(part) return manifest
def __get_or_create_manifest(self, vol_container, file_size): _, bucket, key = self.args['s3_service'].resolve_url_to_location( vol_container['image']['importManifestUrl']) manifest_s3path = '/'.join((bucket, key)) try: with tempfile.SpooledTemporaryFile(max_size=1024000) as \ manifest_destfile: get_req = GetObject.from_other( self, service=self.args['s3_service'], auth=self.args['s3_auth'], source=manifest_s3path, dest=manifest_destfile, show_progress=False) get_req.main() self.log.info('using existing import manifest from the server') manifest_destfile.seek(0) manifest = ImportManifest.read_from_fileobj( manifest_destfile) except ServerError as err: if err.status_code == 404: self.log.info('creating new import manifest') manifest = self.__generate_manifest(vol_container, file_size) tempdir = tempfile.mkdtemp() manifest_filename = os.path.join(tempdir, os.path.basename(key)) with open(manifest_filename, 'w') as manifest_file: manifest.dump_to_fileobj(manifest_file, pretty_print=True) put_req = PutObject.from_other( get_req, source=manifest_filename, dest=manifest_s3path, show_progress=False) put_req.main() os.remove(manifest_filename) os.rmdir(tempdir) else: raise return manifest
def fetch_manifest(self, s3_service, privkey_filename=None): if self.args.get('local_manifest'): _assert_is_file(self.args['local_manifest'], 'manifest') return euca2ools.bundle.manifest.BundleManifest.read_from_file( self.args['local_manifest'], privkey_filename=privkey_filename) # It's on the server, so do things the hard way manifest_s3path = self.get_manifest_s3path() with tempfile.TemporaryFile() as manifest_tempfile: self.log.info('reading manifest from %s', manifest_s3path) req = GetObject.from_other(self, service=s3_service, source=manifest_s3path, dest=manifest_tempfile) try: req.main() except AWSError as err: if err.status_code == 404: self.log.debug('failed to fetch manifest', exc_info=True) raise ValueError("manifest '{0}' does not exist on the " "server".format(manifest_s3path)) raise manifest_tempfile.flush() manifest_tempfile.seek(0) return euca2ools.bundle.manifest.BundleManifest.read_from_fileobj( manifest_tempfile, privkey_filename=privkey_filename)
def main(self): if self.args.get('dry_run'): return if not self.args.get('Image.ImportManifestUrl'): manifest_key = '{0}/{1}.manifest.xml'.format( uuid.uuid4(), self.args['source']) if self.args.get('prefix'): manifest_key = '/'.join((self.args['prefix'], manifest_key)) getobj = GetObject.from_other(self, service=self.args['s3_service'], auth=self.args['s3_auth'], source='/'.join((self.args['bucket'], manifest_key))) days = self.args.get('expires') or 30 expiration = datetime.datetime.utcnow() + datetime.timedelta(days) get_url = getobj.get_presigned_url(expiration) self.log.info('generated manifest GET URL: %s', get_url) self.params['Image.ImportManifestUrl'] = get_url result = self.send() # The manifest creation and uploading parts are done by ResumeImport. if not self.args.get('no_upload'): resume = ResumeImport.from_other( self, source=self.args['source'], task=result['conversionTask']['conversionTaskId'], s3_service=self.args['s3_service'], s3_auth=self.args['s3_auth'], expires=self.args['expires'], show_progress=self.args.get('show_progress', False)) resume.main() return result
def download_bundle_to_fileobj(self, manifest, fileobj, s3_service): # We can skip downloading the manifest since we're just writing all # parts to a file object. parts = self.map_bundle_parts_to_s3paths(manifest) for part, part_s3path in parts: self.log.info('downloading part %s', part_s3path) req = GetObject.from_other( self, service=s3_service, source=part_s3path, dest=fileobj, show_progress=self.args.get('show_progress', False)) response = req.main() self.__check_part_sha1(part, part_s3path, response)
def download_bundle_to_fileobj(self, manifest, fileobj, s3_service): # We can skip downloading the manifest since we're just writing all # parts to a file object. parts = self.map_bundle_parts_to_s3paths(manifest) for part, part_s3path in parts: self.log.info('downloading part %s', part_s3path) req = GetObject.from_other(self, service=s3_service, source=part_s3path, dest=fileobj, show_progress=self.args.get( 'show_progress', False)) response = req.main() self.__check_part_sha1(part, part_s3path, response)
def __download_manifest(self, s3path): with tempfile.SpooledTemporaryFile(max_size=1024000) as \ manifest_destfile: get_req = GetObject.from_other(self, service=self.args['s3_service'], auth=self.args['s3_auth'], source=s3path, dest=manifest_destfile, show_progress=False) try: get_req.main() except AWSError as err: if err.status_code == 404: raise ArgumentError( 'import manifest "{0}" does not exist'.format(s3path)) raise manifest_destfile.seek(0) return ImportManifest.read_from_fileobj(manifest_destfile)
def __download_manifest(self, s3path): with tempfile.SpooledTemporaryFile(max_size=1024000) as manifest_destfile: get_req = GetObject.from_other( self, service=self.args["s3_service"], auth=self.args["s3_auth"], source=s3path, dest=manifest_destfile, show_progress=False, ) try: get_req.main() except AWSError as err: if err.status_code == 404: raise ArgumentError('import manifest "{0}" does not exist'.format(s3path)) raise manifest_destfile.seek(0) return ImportManifest.read_from_fileobj(manifest_destfile)
def main(self): if self.args.get("dry_run"): return if self.args.get("bucket"): self.ensure_bucket_exists(self.args["bucket"]) if not self.args.get("DiskImage.1.Image.ImportManifestUrl"): manifest_key = "{0}/{1}.manifest.xml".format(uuid.uuid4(), self.args["source"]) if self.args.get("prefix"): manifest_key = "/".join((self.args["prefix"], manifest_key)) getobj = GetObject.from_other( self, service=self.args["s3_service"], auth=self.args["s3_auth"], source="/".join((self.args["bucket"], manifest_key)), ) days = self.args.get("expires") or 30 expiration = datetime.datetime.utcnow() + datetime.timedelta(days) get_url = getobj.get_presigned_url(expiration) self.log.info("generated manifest GET URL: %s", get_url) self.params["DiskImage.1.Image.ImportManifestUrl"] = get_url result = self.send() # The manifest creation and uploading parts are done by ResumeImport. if not self.args.get("no_upload"): resume = ResumeImport.from_other( self, source=self.args["source"], task=result["conversionTask"]["conversionTaskId"], s3_service=self.args["s3_service"], s3_auth=self.args["s3_auth"], expires=self.args["expires"], show_progress=self.args.get("show_progress", False), ) resume.main() return result