def _get_manifest_obj(self, private_key=None, dest_dir=None):
'''
Attempt to return a BundleManifest obj based upon the provided
DownloadBundle 'manifest' argument. If the arg is not a BundleManifest
an attempt is made to retrieve the manifest from either a;
local file path, remote bucket/prefix, or fileobj, whichever arg type
is provided.
:param private_key: Local file path to key used to create the bundle.
:dest_dir: Local dir path. If provided, and manifest is to be
downloaded, the manifest will also be written to a file
in this directory.
:returns: BundleManifest obj
'''
if self.args.get('manifest'):
if isinstance(self.args.get('manifest'), BundleManifest):
return self.args.get('manifest')
else:
#Read in manifest from local file path...
manifest = os.path.expanduser(os.path.abspath(
self.args['manifest']))
if not os.path.exists(manifest):
raise ArgumentError("Manifest '{0}' does not exist"
.format(manifest))
if not os.path.isfile(manifest):
raise ArgumentError("Manifest '{0}' is not a file"
.format(manifest))
#Read manifest into BundleManifest obj...
return BundleManifest.read_from_file(manifest, private_key)
else:
return self._download_manifest(bucket=self.args.get('bucket'),
prefix=self.args.get('prefix'),
private_key=private_key,
dest_dir=dest_dir)
def configure(self):
BaseCommand.configure(self)
self.update_config_view()
#Get optional destination directory...
dest_file = self.args['destination']
if not isinstance(dest_file, file) and dest_file != "-":
dest_file = os.path.expanduser(os.path.abspath(dest_file))
self.args['destination'] = dest_file
#Get Mandatory manifest...
manifest = self.args.get('manifest', None)
if manifest:
if not isinstance(manifest, BundleManifest):
#Read manifest file into manifest obj...
manifest_path = os.path.expanduser(os.path.abspath(
self.args['manifest']))
if not os.path.exists(manifest_path):
raise ArgumentError("Manifest '{0}' does not exist"
.format(self.args['manifest']))
if not os.path.isfile(manifest_path):
raise ArgumentError("Manifest '{0}' is not a file"
.format(self.args['manifest']))
#Get the mandatory private key...
if not self.args.get('privatekey'):
config_privatekey = self.config.get_user_option(
'private-key')
if self.args.get('userregion'):
self.args['privatekey'] = config_privatekey
elif 'EC2_PRIVATE_KEY' in os.environ:
self.args['privatekey'] = os.getenv('EC2_PRIVATE_KEY')
elif config_privatekey:
self.args['privatekey'] = config_privatekey
else:
raise ArgumentError(
'missing private key needed to read manifest;'
' please supply one with -k')
privatekey = self.args['privatekey']
self.args['privatekey'] = os.path.expanduser(
os.path.expandvars(privatekey))
if not os.path.exists(self.args['privatekey']):
raise ArgumentError("private key file '{0}' does not exist"
.format(self.args['privatekey']))
if not os.path.isfile(self.args['privatekey']):
raise ArgumentError("private key file '{0}' is not a file"
.format(self.args['privatekey']))
#Read manifest into BundleManifest obj...
manifest = BundleManifest.read_from_file(
manifest_path,
self.args['privatekey'])
self.args['manifest'] = manifest
if not self.args.get('enc_key') and manifest:
self.args['enc_key'] = manifest.enc_key
if not self.args.get('enc_iv') and manifest:
self.args['enc_iv'] = manifest.enc_iv
if not self.args.get('enc_key') or not self.args.get('enc_iv'):
raise ArgumentError('Encryption key (-e) and initialization vector'
' (-v) are required if manifest (-m) is not'
' provided')
def main(self):
key_prefix = self.get_bundle_key_prefix()
self.ensure_dest_bucket_exists()
manifest = BundleManifest.read_from_file(self.args['manifest'])
part_dir = (self.args.get('directory')
or os.path.dirname(self.args['manifest']))
for part in manifest.image_parts:
part.filename = os.path.join(part_dir, part.filename)
if not os.path.isfile(part.filename):
raise ValueError("no such part: '{0}'".format(part.filename))
# manifest -> upload
part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
part_gen = multiprocessing.Process(target=_generate_bundle_parts,
args=(manifest, part_out_w))
part_gen.start()
part_out_w.close()
# Drive the upload process by feeding in part info
self.upload_bundle_parts(part_out_r,
key_prefix,
show_progress=self.args.get('show_progress'))
part_gen.join()
# (conditionally) upload the manifest
if not self.args.get('skipmanifest'):
manifest_dest = (key_prefix +
os.path.basename(self.args['manifest']))
req = PutObject.from_other(self,
source=self.args['manifest'],
dest=manifest_dest,
acl=self.args.get('acl')
or 'aws-exec-read',
retries=self.args.get('retries') or 0)
req.main()
else:
manifest_dest = None
return {
'parts':
tuple({
'filename': part.filename,
'key': (key_prefix + os.path.basename(part.filename))
} for part in manifest.image_parts),
'manifests': ({
'filename': self.args['manifest'],
'key': manifest_dest
}, )
}
def main(self):
key_prefix = self.get_bundle_key_prefix()
self.ensure_dest_bucket_exists()
manifest = BundleManifest.read_from_file(self.args["manifest"])
part_dir = self.args.get("directory") or os.path.dirname(self.args["manifest"])
for part in manifest.image_parts:
part.filename = os.path.join(part_dir, part.filename)
if not os.path.isfile(part.filename):
raise ValueError("no such part: '{0}'".format(part.filename))
# manifest -> upload
part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
part_gen = multiprocessing.Process(target=_generate_bundle_parts, args=(manifest, part_out_w))
part_gen.start()
part_out_w.close()
# Drive the upload process by feeding in part info
self.upload_bundle_parts(part_out_r, key_prefix, show_progress=self.args.get("show_progress"))
part_gen.join()
# (conditionally) upload the manifest
if not self.args.get("skip_manifest"):
manifest_dest = key_prefix + os.path.basename(self.args["manifest"])
req = PutObject(
source=self.args["manifest"],
dest=manifest_dest,
acl=self.args.get("acl") or "aws-exec-read",
retries=self.args.get("retries") or 0,
service=self.service,
config=self.config,
)
req.main()
else:
manifest_dest = None
return {
"parts": tuple(
{"filename": part.filename, "key": (key_prefix + os.path.basename(part.filename))}
for part in manifest.image_parts
),
"manifests": ({"filename": self.args["manifest"], "key": manifest_dest},),
}
def main(self):
key_prefix = self.get_bundle_key_prefix()
self.ensure_dest_bucket_exists()
manifest = BundleManifest.read_from_file(self.args['manifest'])
part_dir = (self.args.get('directory') or
os.path.dirname(self.args['manifest']))
for part in manifest.image_parts:
part.filename = os.path.join(part_dir, part.filename)
if not os.path.isfile(part.filename):
raise ValueError("no such part: '{0}'".format(part.filename))
# manifest -> upload
part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
part_gen = multiprocessing.Process(target=_generate_bundle_parts,
args=(manifest, part_out_w))
part_gen.start()
part_out_w.close()
# Drive the upload process by feeding in part info
self.upload_bundle_parts(part_out_r, key_prefix,
show_progress=self.args.get('show_progress'))
part_gen.join()
# (conditionally) upload the manifest
if not self.args.get('skipmanifest'):
manifest_dest = (key_prefix +
os.path.basename(self.args['manifest']))
req = PutObject.from_other(
self, source=self.args['manifest'], dest=manifest_dest,
acl=self.args.get('acl') or 'aws-exec-read',
retries=self.args.get('retries') or 0)
req.main()
else:
manifest_dest = None
return {'parts': tuple({'filename': part.filename,
'key': (key_prefix +
os.path.basename(part.filename))}
for part in manifest.image_parts),
'manifests': ({'filename': self.args['manifest'],
'key': manifest_dest},)}
