def main(self):
manifest = BundleManifest.read_from_fileobj(
self.args['manifest'], privkey_filename=self.args['privatekey'])
for part in manifest.image_parts:
part_path = os.path.join(self.args['source'], part.filename)
while part_path.startswith('./'):
part_path = part_path[2:]
if os.path.exists(part_path):
part.filename = part_path
else:
raise RuntimeError(
"bundle part '{0}' does not exist; you may need to use "
"-s to specify where to find the bundle's parts"
.format(part_path))
part_reader_out_r, part_reader_out_w = open_pipe_fileobjs()
part_reader = multiprocessing.Process(
target=self.__read_bundle_parts,
args=(manifest, part_reader_out_w))
part_reader.start()
part_reader_out_w.close()
waitpid_in_thread(part_reader.pid)
image_filename = os.path.join(self.args['destination'],
manifest.image_name)
with open(image_filename, 'w') as image:
unbundlestream = UnbundleStream.from_other(
self, source=part_reader_out_r, dest=image,
enc_key=manifest.enc_key, enc_iv=manifest.enc_iv,
image_size=manifest.image_size,
sha1_digest=manifest.image_digest,
show_progress=self.args.get('show_progress', False))
unbundlestream.main()
return image_filename
def configure(self):
BaseCommand.configure(self)
self.update_config_view()
#Get optional destination directory...
dest_file = self.args['destination']
if not isinstance(dest_file, file) and dest_file != "-":
dest_file = os.path.expanduser(os.path.abspath(dest_file))
self.args['destination'] = dest_file
#Get Mandatory manifest...
manifest = self.args.get('manifest', None)
if manifest:
if not isinstance(manifest, BundleManifest):
#Read manifest file into manifest obj...
manifest_path = os.path.expanduser(os.path.abspath(
self.args['manifest']))
if not os.path.exists(manifest_path):
raise ArgumentError("Manifest '{0}' does not exist"
.format(self.args['manifest']))
if not os.path.isfile(manifest_path):
raise ArgumentError("Manifest '{0}' is not a file"
.format(self.args['manifest']))
#Get the mandatory private key...
if not self.args.get('privatekey'):
config_privatekey = self.config.get_user_option(
'private-key')
if self.args.get('userregion'):
self.args['privatekey'] = config_privatekey
elif 'EC2_PRIVATE_KEY' in os.environ:
self.args['privatekey'] = os.getenv('EC2_PRIVATE_KEY')
elif config_privatekey:
self.args['privatekey'] = config_privatekey
else:
raise ArgumentError(
'missing private key needed to read manifest;'
' please supply one with -k')
privatekey = self.args['privatekey']
self.args['privatekey'] = os.path.expanduser(
os.path.expandvars(privatekey))
if not os.path.exists(self.args['privatekey']):
raise ArgumentError("private key file '{0}' does not exist"
.format(self.args['privatekey']))
if not os.path.isfile(self.args['privatekey']):
raise ArgumentError("private key file '{0}' is not a file"
.format(self.args['privatekey']))
#Read manifest into BundleManifest obj...
manifest = BundleManifest.read_from_file(
manifest_path,
self.args['privatekey'])
self.args['manifest'] = manifest
if not self.args.get('enc_key') and manifest:
self.args['enc_key'] = manifest.enc_key
if not self.args.get('enc_iv') and manifest:
self.args['enc_iv'] = manifest.enc_iv
if not self.args.get('enc_key') or not self.args.get('enc_iv'):
raise ArgumentError('Encryption key (-e) and initialization vector'
' (-v) are required if manifest (-m) is not'
' provided')
def _get_manifest_obj(self, private_key=None, dest_dir=None):
'''
Attempt to return a BundleManifest obj based upon the provided
DownloadBundle 'manifest' argument. If the arg is not a BundleManifest
an attempt is made to retrieve the manifest from either a;
local file path, remote bucket/prefix, or fileobj, whichever arg type
is provided.
:param private_key: Local file path to key used to create the bundle.
:dest_dir: Local dir path. If provided, and manifest is to be
downloaded, the manifest will also be written to a file
in this directory.
:returns: BundleManifest obj
'''
if self.args.get('manifest'):
if isinstance(self.args.get('manifest'), BundleManifest):
return self.args.get('manifest')
else:
#Read in manifest from local file path...
manifest = os.path.expanduser(os.path.abspath(
self.args['manifest']))
if not os.path.exists(manifest):
raise ArgumentError("Manifest '{0}' does not exist"
.format(manifest))
if not os.path.isfile(manifest):
raise ArgumentError("Manifest '{0}' is not a file"
.format(manifest))
#Read manifest into BundleManifest obj...
return BundleManifest.read_from_file(manifest, private_key)
else:
return self._download_manifest(bucket=self.args.get('bucket'),
prefix=self.args.get('prefix'),
private_key=private_key,
dest_dir=dest_dir)
def main(self):
key_prefix = self.get_bundle_key_prefix()
self.ensure_dest_bucket_exists()
manifest = BundleManifest.read_from_file(self.args['manifest'])
part_dir = (self.args.get('directory')
or os.path.dirname(self.args['manifest']))
for part in manifest.image_parts:
part.filename = os.path.join(part_dir, part.filename)
if not os.path.isfile(part.filename):
raise ValueError("no such part: '{0}'".format(part.filename))
# manifest -> upload
part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
part_gen = multiprocessing.Process(target=_generate_bundle_parts,
args=(manifest, part_out_w))
part_gen.start()
part_out_w.close()
# Drive the upload process by feeding in part info
self.upload_bundle_parts(part_out_r,
key_prefix,
show_progress=self.args.get('show_progress'))
part_gen.join()
# (conditionally) upload the manifest
if not self.args.get('skipmanifest'):
manifest_dest = (key_prefix +
os.path.basename(self.args['manifest']))
req = PutObject.from_other(self,
source=self.args['manifest'],
dest=manifest_dest,
acl=self.args.get('acl')
or 'aws-exec-read',
retries=self.args.get('retries') or 0)
req.main()
else:
manifest_dest = None
return {
'parts':
tuple({
'filename': part.filename,
'key': (key_prefix + os.path.basename(part.filename))
} for part in manifest.image_parts),
'manifests': ({
'filename': self.args['manifest'],
'key': manifest_dest
}, )
}
def main(self):
key_prefix = self.get_bundle_key_prefix()
self.ensure_dest_bucket_exists()
manifest = BundleManifest.read_from_file(self.args["manifest"])
part_dir = self.args.get("directory") or os.path.dirname(self.args["manifest"])
for part in manifest.image_parts:
part.filename = os.path.join(part_dir, part.filename)
if not os.path.isfile(part.filename):
raise ValueError("no such part: '{0}'".format(part.filename))
# manifest -> upload
part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
part_gen = multiprocessing.Process(target=_generate_bundle_parts, args=(manifest, part_out_w))
part_gen.start()
part_out_w.close()
# Drive the upload process by feeding in part info
self.upload_bundle_parts(part_out_r, key_prefix, show_progress=self.args.get("show_progress"))
part_gen.join()
# (conditionally) upload the manifest
if not self.args.get("skip_manifest"):
manifest_dest = key_prefix + os.path.basename(self.args["manifest"])
req = PutObject(
source=self.args["manifest"],
dest=manifest_dest,
acl=self.args.get("acl") or "aws-exec-read",
retries=self.args.get("retries") or 0,
service=self.service,
config=self.config,
)
req.main()
else:
manifest_dest = None
return {
"parts": tuple(
{"filename": part.filename, "key": (key_prefix + os.path.basename(part.filename))}
for part in manifest.image_parts
),
"manifests": ({"filename": self.args["manifest"], "key": manifest_dest},),
}
def main(self):
key_prefix = self.get_bundle_key_prefix()
self.ensure_dest_bucket_exists()
manifest = BundleManifest.read_from_file(self.args['manifest'])
part_dir = (self.args.get('directory') or
os.path.dirname(self.args['manifest']))
for part in manifest.image_parts:
part.filename = os.path.join(part_dir, part.filename)
if not os.path.isfile(part.filename):
raise ValueError("no such part: '{0}'".format(part.filename))
# manifest -> upload
part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
part_gen = multiprocessing.Process(target=_generate_bundle_parts,
args=(manifest, part_out_w))
part_gen.start()
part_out_w.close()
# Drive the upload process by feeding in part info
self.upload_bundle_parts(part_out_r, key_prefix,
show_progress=self.args.get('show_progress'))
part_gen.join()
# (conditionally) upload the manifest
if not self.args.get('skipmanifest'):
manifest_dest = (key_prefix +
os.path.basename(self.args['manifest']))
req = PutObject.from_other(
self, source=self.args['manifest'], dest=manifest_dest,
acl=self.args.get('acl') or 'aws-exec-read',
retries=self.args.get('retries') or 0)
req.main()
else:
manifest_dest = None
return {'parts': tuple({'filename': part.filename,
'key': (key_prefix +
os.path.basename(part.filename))}
for part in manifest.image_parts),
'manifests': ({'filename': self.args['manifest'],
'key': manifest_dest},)}
def _download_manifest(self,
bucket=None,
prefix=None,
private_key=None,
dest_dir=None):
'''
Read in bundle manifest from remote bucket.
:param bucket: Bucket to download bundle manifest from
:param prefix: Manifest prefix used to download bundle
:param private_key: Local file path to key used to create the bundle.
:dest_dir: Local dir path. If provided the downloaded manifest will
be written to a file in this directory, otherwise it will
not be written to a local file.
:returns: BundleManifest obj
'''
self.log.debug('download to dest_dir:' + str(dest_dir))
bucket = bucket or self.args.get('bucket')
prefix = prefix or self.args.get('prefix')
if bucket is None or prefix is None:
raise ArgumentError('Need to provide both bucket and prefix'
' to download manifest')
#Make sure the manifest exists, and the prefix is unique...
self.log.debug('Downloading manifest bucket "{0}" prefix "{1}"'
.format(bucket, prefix))
manifest_keys = get_manifest_keys(bucket,
prefix,
service=self.service,
config=self.config)
if not manifest_keys:
if prefix:
raise ArgumentError("no manifests found with prefix '{0}' "
"in bucket '{1}'."
.format(prefix, bucket))
else:
raise ArgumentError("no manifests found in bucket '{0}'."
.format(bucket))
if len(manifest_keys) > 1:
raise RuntimeError('Multiple manifests found:{0}'
.format(",".join(str(m)
for m in manifest_keys)))
manifest_key = manifest_keys.pop()
#Write to a local file if dest_dir was provided...
if dest_dir:
#Download paths to individual files under provided directory...
if not os.path.isdir(dest_dir):
self.log.debug('Creating dir at: {0}'.format(dest_dir))
os.mkdir(dest_dir)
local_file_path = os.path.join(dest_dir,
os.path.basename(manifest_key))
self.log.debug('Writing manifest to "{0}"'.format(local_file_path))
manifest_fileobj = open(local_file_path, 'w+')
else:
manifest_fileobj = BytesIO()
#Read manifest in from fileobj and create BundleManifest obj...
with manifest_fileobj:
try:
path = os.path.join(bucket, manifest_key)
self.log.debug('Attempting to download:{0}'.format(path))
GetObject(paths=[path],
fileobj=manifest_fileobj,
opath=None,
service=self.service,
config=self.config).main()
except AWSError as err:
if err.code != 'NoSuchEntity':
raise
raise ArgumentError(
"cannot find manifest file(s) {0} in bucket '{1}'."
.format(",".join(manifest_keys), bucket))
#Read manifest info from pipe...
manifest_fileobj.seek(0)
manifest = BundleManifest.read_from_fileobj(
manifest_fileobj, privkey_filename=private_key)
manifest.manifest_key = manifest_key
self.log.debug('Returning Manifest for image:{0}'
.format(str(manifest.image_name)))
return manifest