예제 #1
0
 def __get_or_create_manifest(self, vol_container, file_size):
     _, bucket, key = self.args['s3_service'].resolve_url_to_location(
         vol_container['image']['importManifestUrl'])
     manifest_s3path = '/'.join((bucket, key))
     try:
         with tempfile.SpooledTemporaryFile(max_size=1024000) as \
                 manifest_destfile:
             get_req = GetObject.from_other(
                 self, service=self.args['s3_service'],
                 auth=self.args['s3_auth'], source=manifest_s3path,
                 dest=manifest_destfile, show_progress=False)
             get_req.main()
             self.log.info('using existing import manifest from the server')
             manifest_destfile.seek(0)
             manifest = ImportManifest.read_from_fileobj(
                 manifest_destfile)
     except ServerError as err:
         if err.status_code == 404:
             self.log.info('creating new import manifest')
             manifest = self.__generate_manifest(vol_container, file_size)
             tempdir = tempfile.mkdtemp()
             manifest_filename = os.path.join(tempdir,
                                              os.path.basename(key))
             with open(manifest_filename, 'w') as manifest_file:
                 manifest.dump_to_fileobj(manifest_file, pretty_print=True)
             put_req = PutObject.from_other(
                 get_req, source=manifest_filename, dest=manifest_s3path,
                 show_progress=False)
             put_req.main()
             os.remove(manifest_filename)
             os.rmdir(tempdir)
         else:
             raise
     return manifest
예제 #2
0
파일: mixins.py 프로젝트: Debian/euca2ools
 def upload_bundle_file(self, source, dest, show_progress=False,
                        **putobj_kwargs):
     if self.args.get('upload_policy'):
         if show_progress:
             # PostObject does not yet support show_progress
             print source, 'uploading...'
         if self.args.get('security_token'):
             postobj_kwargs = \
                 {'x-amz-security-token': self.args['security_token']}
         else:
             postobj_kwargs = {}
         postobj_kwargs.update(putobj_kwargs)
         req = PostObject.from_other(
             self, source=source, dest=dest,
             acl=self.args.get('acl') or 'aws-exec-read',
             Policy=self.args['upload_policy'],
             Signature=self.args['upload_policy_signature'],
             AWSAccessKeyId=self.args['key_id'], **postobj_kwargs)
     else:
         req = PutObject.from_other(
             self, source=source, dest=dest,
             acl=self.args.get('acl') or 'aws-exec-read',
             retries=self.args.get('retries') or 0,
             show_progress=show_progress, **putobj_kwargs)
     req.main()
예제 #3
0
 def upload_bundle_file(self,
                        source,
                        dest,
                        show_progress=False,
                        **putobj_kwargs):
     if self.args.get('upload_policy'):
         if show_progress:
             # PostObject does not yet support show_progress
             print source, 'uploading...'
         req = PostObject.from_other(
             self,
             source=source,
             dest=dest,
             acl=self.args.get('acl') or 'aws-exec-read',
             Policy=self.args['upload_policy'],
             Signature=self.args['upload_policy_signature'],
             AWSAccessKeyId=self.args['key_id'],
             **putobj_kwargs)
     else:
         req = PutObject.from_other(self,
                                    source=source,
                                    dest=dest,
                                    acl=self.args.get('acl')
                                    or 'aws-exec-read',
                                    retries=self.args.get('retries') or 0,
                                    show_progress=show_progress,
                                    **putobj_kwargs)
     req.main()
예제 #4
0
    def main(self):
        key_prefix = self.get_bundle_key_prefix()
        self.ensure_dest_bucket_exists()

        manifest = BundleManifest.read_from_file(self.args["manifest"])
        part_dir = self.args.get("directory") or os.path.dirname(self.args["manifest"])
        for part in manifest.image_parts:
            part.filename = os.path.join(part_dir, part.filename)
            if not os.path.isfile(part.filename):
                raise ValueError("no such part: '{0}'".format(part.filename))

        # manifest -> upload
        part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
        part_gen = multiprocessing.Process(target=_generate_bundle_parts, args=(manifest, part_out_w))
        part_gen.start()
        part_out_w.close()

        # Drive the upload process by feeding in part info
        self.upload_bundle_parts(part_out_r, key_prefix, show_progress=self.args.get("show_progress"))
        part_gen.join()

        # (conditionally) upload the manifest
        if not self.args.get("skip_manifest"):
            manifest_dest = key_prefix + os.path.basename(self.args["manifest"])
            req = PutObject(
                source=self.args["manifest"],
                dest=manifest_dest,
                acl=self.args.get("acl") or "aws-exec-read",
                retries=self.args.get("retries") or 0,
                service=self.service,
                config=self.config,
            )
            req.main()
        else:
            manifest_dest = None

        return {
            "parts": tuple(
                {"filename": part.filename, "key": (key_prefix + os.path.basename(part.filename))}
                for part in manifest.image_parts
            ),
            "manifests": ({"filename": self.args["manifest"], "key": manifest_dest},),
        }
예제 #5
0
    def main(self):
        key_prefix = self.get_bundle_key_prefix()
        self.ensure_dest_bucket_exists()

        manifest = BundleManifest.read_from_file(self.args['manifest'])
        part_dir = (self.args.get('directory') or
                    os.path.dirname(self.args['manifest']))
        for part in manifest.image_parts:
            part.filename = os.path.join(part_dir, part.filename)
            if not os.path.isfile(part.filename):
                raise ValueError("no such part: '{0}'".format(part.filename))

        # manifest -> upload
        part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
        part_gen = multiprocessing.Process(target=_generate_bundle_parts,
                                           args=(manifest, part_out_w))
        part_gen.start()
        part_out_w.close()

        # Drive the upload process by feeding in part info
        self.upload_bundle_parts(part_out_r, key_prefix,
                                 show_progress=self.args.get('show_progress'))
        part_gen.join()

        # (conditionally) upload the manifest
        if not self.args.get('skip_manifest'):
            manifest_dest = (key_prefix +
                             os.path.basename(self.args['manifest']))
            req = PutObject(source=self.args['manifest'], dest=manifest_dest,
                            acl=self.args.get('acl') or 'aws-exec-read',
                            retries=self.args.get('retries') or 0,
                            service=self.service, config=self.config)
            req.main()
        else:
            manifest_dest = None

        return {'parts': tuple({'filename': part.filename,
                                'key': (key_prefix +
                                        os.path.basename(part.filename))}
                               for part in manifest.image_parts),
                'manifests': ({'filename': self.args['manifest'],
                               'key': manifest_dest},)}
예제 #6
0
 def __upload_part(self, part, part_s3path, pbar_label_template):
     self.log.info('Uploading part %s (bytes %i-%i)', part_s3path,
                   part.start, part.end)
     part_pbar_label = pbar_label_template.format(
         fname=os.path.basename(part.key), index=(part.index + 1))
     with open(self.args['source']) as source:
         source.seek(part.start)
         put_req = PutObject.from_other(
             self, service=self.args['s3_service'],
             auth=self.args['s3_auth'], source=source, dest=part_s3path,
             size=(part.end - part.start + 1),
             show_progress=self.args.get('show_progress', False),
             progressbar_label=part_pbar_label)
         return put_req.main()
예제 #7
0
    def main(self):
        key_prefix = self.get_bundle_key_prefix()
        self.ensure_dest_bucket_exists()

        manifest = BundleManifest.read_from_file(self.args['manifest'])
        part_dir = (self.args.get('directory') or
                    os.path.dirname(self.args['manifest']))
        for part in manifest.image_parts:
            part.filename = os.path.join(part_dir, part.filename)
            if not os.path.isfile(part.filename):
                raise ValueError("no such part: '{0}'".format(part.filename))

        # manifest -> upload
        part_out_r, part_out_w = multiprocessing.Pipe(duplex=False)
        part_gen = multiprocessing.Process(target=_generate_bundle_parts,
                                           args=(manifest, part_out_w))
        part_gen.start()
        part_out_w.close()

        # Drive the upload process by feeding in part info
        self.upload_bundle_parts(part_out_r, key_prefix,
                                 show_progress=self.args.get('show_progress'))
        part_gen.join()

        # (conditionally) upload the manifest
        if not self.args.get('skipmanifest'):
            manifest_dest = (key_prefix +
                             os.path.basename(self.args['manifest']))
            req = PutObject.from_other(
                self, source=self.args['manifest'], dest=manifest_dest,
                acl=self.args.get('acl') or 'aws-exec-read',
                retries=self.args.get('retries') or 0)
            req.main()
        else:
            manifest_dest = None

        return {'parts': tuple({'filename': part.filename,
                                'key': (key_prefix +
                                        os.path.basename(part.filename))}
                               for part in manifest.image_parts),
                'manifests': ({'filename': self.args['manifest'],
                               'key': manifest_dest},)}