Esempio n. 1
0
    def main(self):
        manifest = BundleManifest.read_from_fileobj(
            self.args['manifest'], privkey_filename=self.args['privatekey'])

        for part in manifest.image_parts:
            part_path = os.path.join(self.args['source'], part.filename)
            while part_path.startswith('./'):
                part_path = part_path[2:]
            if os.path.exists(part_path):
                part.filename = part_path
            else:
                raise RuntimeError(
                    "bundle part '{0}' does not exist; you may need to use "
                    "-s to specify where to find the bundle's parts"
                    .format(part_path))

        part_reader_out_r, part_reader_out_w = open_pipe_fileobjs()
        part_reader = multiprocessing.Process(
            target=self.__read_bundle_parts,
            args=(manifest, part_reader_out_w))
        part_reader.start()
        part_reader_out_w.close()
        waitpid_in_thread(part_reader.pid)

        image_filename = os.path.join(self.args['destination'],
                                      manifest.image_name)
        with open(image_filename, 'w') as image:
            unbundlestream = UnbundleStream.from_other(
                self, source=part_reader_out_r, dest=image,
                enc_key=manifest.enc_key, enc_iv=manifest.enc_iv,
                image_size=manifest.image_size,
                sha1_digest=manifest.image_digest,
                show_progress=self.args.get('show_progress', False))
            unbundlestream.main()
        return image_filename
Esempio n. 2
0
    def configure(self):
        BaseCommand.configure(self)
        self.update_config_view()

        #Get optional destination directory...
        dest_file = self.args['destination']
        if not isinstance(dest_file, file) and dest_file != "-":
            dest_file = os.path.expanduser(os.path.abspath(dest_file))
            self.args['destination'] = dest_file

        #Get Mandatory manifest...
        manifest = self.args.get('manifest', None)
        if manifest:
            if not isinstance(manifest, BundleManifest):
                #Read manifest file into manifest obj...
                manifest_path = os.path.expanduser(os.path.abspath(
                    self.args['manifest']))
                if not os.path.exists(manifest_path):
                    raise ArgumentError("Manifest '{0}' does not exist"
                                        .format(self.args['manifest']))
                if not os.path.isfile(manifest_path):
                    raise ArgumentError("Manifest '{0}' is not a file"
                                        .format(self.args['manifest']))
                #Get the mandatory private key...
                if not self.args.get('privatekey'):
                    config_privatekey = self.config.get_user_option(
                        'private-key')
                    if self.args.get('userregion'):
                        self.args['privatekey'] = config_privatekey
                    elif 'EC2_PRIVATE_KEY' in os.environ:
                        self.args['privatekey'] = os.getenv('EC2_PRIVATE_KEY')
                    elif config_privatekey:
                        self.args['privatekey'] = config_privatekey
                    else:
                        raise ArgumentError(
                            'missing private key needed to read manifest;'
                            ' please supply one with -k')
                privatekey = self.args['privatekey']
                self.args['privatekey'] = os.path.expanduser(
                    os.path.expandvars(privatekey))
                if not os.path.exists(self.args['privatekey']):
                    raise ArgumentError("private key file '{0}' does not exist"
                                        .format(self.args['privatekey']))
                if not os.path.isfile(self.args['privatekey']):
                    raise ArgumentError("private key file '{0}' is not a file"
                                        .format(self.args['privatekey']))
                #Read manifest into BundleManifest obj...
                manifest = BundleManifest.read_from_file(
                    manifest_path,
                    self.args['privatekey'])
                self.args['manifest'] = manifest
            if not self.args.get('enc_key') and manifest:
                self.args['enc_key'] = manifest.enc_key
            if not self.args.get('enc_iv') and manifest:
                self.args['enc_iv'] = manifest.enc_iv

        if not self.args.get('enc_key') or not self.args.get('enc_iv'):
            raise ArgumentError('Encryption key (-e) and initialization vector'
                                ' (-v) are required if manifest (-m) is not'
                                ' provided')
Esempio n. 3
0
 def _get_manifest_obj(self, private_key=None, dest_dir=None):
     '''
     Attempt to return a BundleManifest obj based upon the provided
     DownloadBundle 'manifest' argument. If the arg is not a BundleManifest
     an attempt is made to retrieve the manifest from either a;
     local file path, remote bucket/prefix, or fileobj, whichever arg type
     is provided.
     :param private_key: Local file path to key used to create the bundle.
     :dest_dir: Local dir path. If provided, and manifest is to be
                downloaded, the manifest will also be written to a file
                in this directory.
     :returns: BundleManifest obj
     '''
     if self.args.get('manifest'):
         if isinstance(self.args.get('manifest'), BundleManifest):
             return self.args.get('manifest')
         else:
             #Read in manifest from local file path...
             manifest = os.path.expanduser(os.path.abspath(
                 self.args['manifest']))
             if not os.path.exists(manifest):
                 raise ArgumentError("Manifest '{0}' does not exist"
                                     .format(manifest))
             if not os.path.isfile(manifest):
                 raise ArgumentError("Manifest '{0}' is not a file"
                                     .format(manifest))
             #Read manifest into BundleManifest obj...
             return BundleManifest.read_from_file(manifest, private_key)
     else:
         return self._download_manifest(bucket=self.args.get('bucket'),
                                        prefix=self.args.get('prefix'),
                                        private_key=private_key,
                                        dest_dir=dest_dir)
Esempio n. 4
0
    def main(self):
        key_prefix = self.get_bundle_key_prefix()
        self.ensure_dest_bucket_exists()

        manifest = BundleManifest.read_from_file(self.args['manifest'])
        part_dir = (self.args.get('directory')
                    or os.path.dirname(self.args['manifest']))
        for part in manifest.image_parts:
            part.filename = os.path.join(part_dir, part.filename)
            if not os.path.isfile(part.filename):
                raise ValueError("no such part: '{0}'".format(part.filename))

        # manifest -> upload
        part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
        part_gen = multiprocessing.Process(target=_generate_bundle_parts,
                                           args=(manifest, part_out_w))
        part_gen.start()
        part_out_w.close()

        # Drive the upload process by feeding in part info
        self.upload_bundle_parts(part_out_r,
                                 key_prefix,
                                 show_progress=self.args.get('show_progress'))
        part_gen.join()

        # (conditionally) upload the manifest
        if not self.args.get('skipmanifest'):
            manifest_dest = (key_prefix +
                             os.path.basename(self.args['manifest']))
            req = PutObject.from_other(self,
                                       source=self.args['manifest'],
                                       dest=manifest_dest,
                                       acl=self.args.get('acl')
                                       or 'aws-exec-read',
                                       retries=self.args.get('retries') or 0)
            req.main()
        else:
            manifest_dest = None

        return {
            'parts':
            tuple({
                'filename': part.filename,
                'key': (key_prefix + os.path.basename(part.filename))
            } for part in manifest.image_parts),
            'manifests': ({
                'filename': self.args['manifest'],
                'key': manifest_dest
            }, )
        }
Esempio n. 5
0
    def main(self):
        key_prefix = self.get_bundle_key_prefix()
        self.ensure_dest_bucket_exists()

        manifest = BundleManifest.read_from_file(self.args["manifest"])
        part_dir = self.args.get("directory") or os.path.dirname(self.args["manifest"])
        for part in manifest.image_parts:
            part.filename = os.path.join(part_dir, part.filename)
            if not os.path.isfile(part.filename):
                raise ValueError("no such part: '{0}'".format(part.filename))

        # manifest -> upload
        part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
        part_gen = multiprocessing.Process(target=_generate_bundle_parts, args=(manifest, part_out_w))
        part_gen.start()
        part_out_w.close()

        # Drive the upload process by feeding in part info
        self.upload_bundle_parts(part_out_r, key_prefix, show_progress=self.args.get("show_progress"))
        part_gen.join()

        # (conditionally) upload the manifest
        if not self.args.get("skip_manifest"):
            manifest_dest = key_prefix + os.path.basename(self.args["manifest"])
            req = PutObject(
                source=self.args["manifest"],
                dest=manifest_dest,
                acl=self.args.get("acl") or "aws-exec-read",
                retries=self.args.get("retries") or 0,
                service=self.service,
                config=self.config,
            )
            req.main()
        else:
            manifest_dest = None

        return {
            "parts": tuple(
                {"filename": part.filename, "key": (key_prefix + os.path.basename(part.filename))}
                for part in manifest.image_parts
            ),
            "manifests": ({"filename": self.args["manifest"], "key": manifest_dest},),
        }
Esempio n. 6
0
    def main(self):
        key_prefix = self.get_bundle_key_prefix()
        self.ensure_dest_bucket_exists()

        manifest = BundleManifest.read_from_file(self.args['manifest'])
        part_dir = (self.args.get('directory') or
                    os.path.dirname(self.args['manifest']))
        for part in manifest.image_parts:
            part.filename = os.path.join(part_dir, part.filename)
            if not os.path.isfile(part.filename):
                raise ValueError("no such part: '{0}'".format(part.filename))

        # manifest -> upload
        part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
        part_gen = multiprocessing.Process(target=_generate_bundle_parts,
                                           args=(manifest, part_out_w))
        part_gen.start()
        part_out_w.close()

        # Drive the upload process by feeding in part info
        self.upload_bundle_parts(part_out_r, key_prefix,
                                 show_progress=self.args.get('show_progress'))
        part_gen.join()

        # (conditionally) upload the manifest
        if not self.args.get('skipmanifest'):
            manifest_dest = (key_prefix +
                             os.path.basename(self.args['manifest']))
            req = PutObject.from_other(
                self, source=self.args['manifest'], dest=manifest_dest,
                acl=self.args.get('acl') or 'aws-exec-read',
                retries=self.args.get('retries') or 0)
            req.main()
        else:
            manifest_dest = None

        return {'parts': tuple({'filename': part.filename,
                                'key': (key_prefix +
                                        os.path.basename(part.filename))}
                               for part in manifest.image_parts),
                'manifests': ({'filename': self.args['manifest'],
                               'key': manifest_dest},)}
Esempio n. 7
0
 def _download_manifest(self,
                        bucket=None,
                        prefix=None,
                        private_key=None,
                        dest_dir=None):
     '''
     Read in bundle manifest from remote bucket.
     :param bucket: Bucket to download bundle manifest from
     :param prefix: Manifest prefix used to download bundle
     :param private_key: Local file path to key used to create the bundle.
     :dest_dir: Local dir path. If provided the downloaded manifest will
                be written to a file in this directory, otherwise it will
                not be written to a local file.
     :returns: BundleManifest obj
     '''
     self.log.debug('download to dest_dir:' + str(dest_dir))
     bucket = bucket or self.args.get('bucket')
     prefix = prefix or self.args.get('prefix')
     if bucket is None or prefix is None:
         raise ArgumentError('Need to provide both bucket and prefix'
                             ' to download manifest')
     #Make sure the manifest exists, and the prefix is unique...
     self.log.debug('Downloading manifest bucket "{0}" prefix "{1}"'
                    .format(bucket, prefix))
     manifest_keys = get_manifest_keys(bucket,
                                       prefix,
                                       service=self.service,
                                       config=self.config)
     if not manifest_keys:
         if prefix:
             raise ArgumentError("no manifests found with prefix '{0}' "
                                 "in bucket '{1}'."
                                 .format(prefix, bucket))
         else:
             raise ArgumentError("no manifests found in bucket '{0}'."
                                 .format(bucket))
     if len(manifest_keys) > 1:
         raise RuntimeError('Multiple manifests found:{0}'
                            .format(",".join(str(m)
                                    for m in manifest_keys)))
     manifest_key = manifest_keys.pop()
     #Write to a local file if dest_dir was provided...
     if dest_dir:
         #Download paths to individual files under provided directory...
         if not os.path.isdir(dest_dir):
             self.log.debug('Creating dir at: {0}'.format(dest_dir))
             os.mkdir(dest_dir)
         local_file_path = os.path.join(dest_dir,
                                        os.path.basename(manifest_key))
         self.log.debug('Writing manifest to "{0}"'.format(local_file_path))
         manifest_fileobj = open(local_file_path, 'w+')
     else:
         manifest_fileobj = BytesIO()
     #Read manifest in from fileobj and create BundleManifest obj...
     with manifest_fileobj:
         try:
             path = os.path.join(bucket, manifest_key)
             self.log.debug('Attempting to download:{0}'.format(path))
             GetObject(paths=[path],
                       fileobj=manifest_fileobj,
                       opath=None,
                       service=self.service,
                       config=self.config).main()
         except AWSError as err:
             if err.code != 'NoSuchEntity':
                 raise
             raise ArgumentError(
                 "cannot find manifest file(s) {0} in bucket '{1}'."
                 .format(",".join(manifest_keys), bucket))
         #Read manifest info from pipe...
         manifest_fileobj.seek(0)
         manifest = BundleManifest.read_from_fileobj(
             manifest_fileobj, privkey_filename=private_key)
     manifest.manifest_key = manifest_key
     self.log.debug('Returning Manifest for image:{0}'
                    .format(str(manifest.image_name)))
     return manifest