Пример #1
0
class InstallImage(EuStoreRequest, FileTransferProgressBarMixin):
    DESCRIPTION = 'Download an image from EuStore and add it to your cloud'
    ARGS = [MutuallyExclusiveArgList(True,
                Arg('-i', '--image-name', metavar='EUIMAGE',
                    help='name of the image to download and install'),
                Arg('-t', '--tarball', metavar='FILE',
                    help='tarball to install the image from')),
            Arg('-b', '--bucket', required=True,
                help='bucket to store the images in (required)'),
            Arg('-s', '--description', metavar='DESC',
                help='image description (required for -t)'),
            Arg('-a', '--architecture', choices=('i386', 'x86_64', 'armhf'),
                help='image architecture (required for -t)'),
            Arg('-p', '--prefix', help='prefix to use when naming the image'),
            Arg('--hypervisor', choices=('xen', 'kvm', 'universal'),
                help='''hypervisor the kernel image is built for (required for
                images with hypervisor-specific kernels'''),
            Arg('--separate-buckets', action='store_true',
                help='''store kernel, ramdisk, and machine images in separate
                buckets (BUCKET-kernel, BUCKET-ramdisk)'''),
            Arg('-k', '--kernel-type', dest='kernel_type',
                choices=('xen', 'kvm', 'universal'), help=argparse.SUPPRESS),
            Arg('-d', '--directory', dest='directory', metavar='DIR',
                help='''location to place the image and other artifacts
                (default:  dir named by TMPDIR, TEMP, or TMP environment
                variables, or otherwise /var/tmp)'''),
            Arg('--pad-name', action='store_true', help='''add some random
                characters to the image's name to ensure it is unique'''),
            Arg('--kernel', help='''ID of the kernel image to use instead of
                the one bundled with the image'''),
            Arg('--ramdisk', help='''ID of the ramdisk image to use instead of
                the one bundled with the image'''),
            Arg('-I', '--access-key-id', dest='key_id', metavar='KEY_ID'),
            Arg('-S', '--secret-key', dest='secret_key', metavar='KEY'),
            Arg('-c', '--cert', metavar='FILE',
                help='file containing your signing certificate'),
            Arg('--privatekey', metavar='FILE', help='''file containing the
                private key to sign the bundle's manifest with.  This private
                key will also be required to unbundle the image in the
                future.'''),
            Arg('--ec2cert', metavar='FILE', help='''file containing the
                cloud's X.509 certificate'''),
            Arg('-u', '--user', metavar='ACCOUNT', help='your account ID'),
            Arg('--ec2-url', metavar='URL',
                help='compute service endpoint URL'),
            Arg('--iam-url', metavar='URL',
                help='identity service endpoint URL'),
            Arg('--s3-url', metavar='URL',
                help='storage service endpoint URL'),
            Arg('-y', '--yes', action='store_true', help=argparse.SUPPRESS)]

    # noinspection PyExceptionInherit
    def configure(self):
        EuStoreRequest.configure(self)
        set_userregion(self.config, self.args.get('userregion'))

        if self.args.get('kernel_type'):
            # Use it and complain
            self.args['hypervisor'] = self.args['kernel_type']
            msg = ('argument -k/--kernel-type is deprecated; use --hypervisor '
                   'instead')
            self.log.warn(msg)
            print >> sys.stderr, 'warning:', msg

        # Get bundle creds first
        add_bundle_creds(self.args, self.config)
        if not self.args.get('cert'):
            raise ArgumentError(
                'missing certificate; please supply one with -c')
        self.log.debug('certificate: %s', self.args['cert'])
        if not self.args.get('privatekey'):
            raise ArgumentError(
                'missing private key; please supply one with --privatekey')
        self.log.debug('private key: %s', self.args['privatekey'])
        if not self.args.get('ec2cert'):
            raise ArgumentError(
                'missing cloud certificate; please supply one with --ec2cert')
        self.log.debug('cloud certificate: %s', self.args['ec2cert'])
        if not self.args.get('user'):
            raise ArgumentError(
                'missing account ID; please supply one with --user')
        self.log.debug('account ID: %s', self.args['user'])

        # Set up the web services -- we're going to use them a lot
        query_auth = QuerySigV2Auth(self.config,
                                    key_id=self.args.get('key_id'),
                                    secret_key=self.args.get('secret_key'))
        self.__euare = Euare(self.config, loglevel=self.log.level,
                             auth=copy.copy(query_auth),
                             url=self.args.get('iam_url'))
        self.log.debug('configuring euare service')
        self.__euare.configure()
        self.__eucalyptus = Eucalyptus(self.config, loglevel=self.log.level,
                                       auth=copy.copy(query_auth),
                                       url=self.args.get('ec2_url'))
        self.log.debug('configuring eucalyptus service')
        self.__eucalyptus.configure()
        s3_auth = S3RestAuth(self.config, key_id=self.args.get('key_id'),
                             secret_key=self.args.get('secret_key'))
        self.__walrus = Walrus(self.config, loglevel=self.log.level,
                               auth=s3_auth, url=self.args.get('s3_url'))
        self.__walrus.configure()

        # Check other args next
        if self.args.get('tarball'):
            if not self.args.get('architecture'):
                raise ArgumentError('argument -a/--architecture is required '
                                    'when -t/--tarball is used')
            if not self.args.get('hypervisor'):
                raise ArgumentError('argument --hypervisor is required when '
                                    '-t/--tarball is used')
            self.args['tarball'] = os.path.expanduser(os.path.expandvars(
                self.args['tarball']))
            if not os.path.exists(self.args['tarball']):
                raise ArgumentError("tarball file '{0}' does not exist"
                                    .format(self.args['tarball']))
            if not os.path.isfile(self.args['tarball']):
                raise ArgumentError("tarball file '{0}' is not a file"
                                    .format(self.args['tarball']))
        if self.args.get('kernel') and not self.args.get('ramdisk'):
            raise ArgumentError('argument --kernel: --ramdisk is required')
        if self.args.get('ramdisk') and not self.args.get('kernel'):
            raise ArgumentError('argument --ramdisk: --kernel is required')
        if self.args.get('image') and self.args.get('architecture'):
            self.log.warn("downloaded image's architecture may be overridden")

    def ensure_kernel_reg_privs(self):
        req = ListAccountAliases(service=self.__euare, config=self.config)
        response = req.main()
        for alias in response.get('AccountAliases', []):
            if alias == 'eucalyptus':
                self.log.debug("found account alias '%s'; ok to register "
                               "kernel/ramdisk images", alias)
                return
        raise ClientError("kernel/ramdisk images may only be registered by "
                          "the 'eucalyptus' account")

    def main(self):
        if not self.args.get('kernel') or not self.args.get('ramdisk'):
            self.ensure_kernel_reg_privs()

        if self.args.get('directory'):
            workdir = self.args['directory']
            should_delete_workdir = False
        else:
            workdir = mkdtemp_for_large_files()
            self.log.debug('created working directory %s', workdir)
            should_delete_workdir = True

        tarball_path = self.get_tarball(workdir=workdir)
        image_ids = self.bundle_and_register_all(workdir, tarball_path)
        if should_delete_workdir:
            shutil.rmtree(workdir)
        return image_ids

    def print_result(self, image_ids):
        print 'Installed new image', image_ids['machine']

    def get_tarball(self, workdir):
        if self.args.get('tarball'):
            self.log.info('using local tarball %s', self.args['tarball'])
            return self.args['tarball']
        else:
            # Download one
            req = euca2ools.commands.eustore.describeimages.DescribeImages(
                service=self.service, config=self.config)
            eustore_images = req.main()
            for image in eustore_images.get('images', []):
                if image.get('name') == self.args['image_name']:
                    break
            else:
                raise KeyError("no such image: '{0}'"
                               .format(self.args['image_name']))
            # pylint: disable=W0631
            self.log.debug('image data: %s', str(image))
            if self.args.get('architecture') is None:
                self.args['architecture'] = image.get('architecture')
            if self.args.get('description') is None:
                self.args['description'] = image.get('description')
            if bool(image.get('single-kernel', False)):
                self.log.debug('image catalog data specify single-kernel; '
                               "setting hypervisor to 'universal'")
                self.args['hypervisor'] = 'universal'
            if not self.args.get('hypervisor'):
                raise RuntimeError("image '{0}' uses hypervisor-specific "
                                   "kernels; please specify a hypervisor with "
                                   "--hypervisor"
                                   .format(self.args['image_name']))
            if self.service.endpoint.endswith('/'):
                endpoint = self.service.endpoint
            else:
                endpoint = self.service.endpoint + '/'
            url = urlparse.urljoin(endpoint, image['url'])
            self.log.info('downloading image from %s', url)
            label = 'Downloading image '.format(os.path.basename(url))
            req = requestbuilder.commands.http.Get(
                label=label, url=url,
                show_progress=self.args.get('show_progress', False),
                dest=workdir, config=self.config)
            tarball_path, tarball_size = req.main()
            self.log.info('downloaded %i bytes to %s', tarball_size,
                          tarball_path)

            expected_crc = image['name']  # Yes, really.
            real_crc = self.calc_file_checksum(tarball_path)
            if real_crc != expected_crc:
                raise RuntimeError('downloaded file is incomplete or corrupt '
                                   '(checksum: {0}, expected: {1})'
                                   .format(real_crc, expected_crc))
            return tarball_path
            # pylint: enable=W0631

    def calc_file_checksum(self, filename):
        filesize = os.path.getsize(filename)
        pbar = self.get_progressbar(label='Verifying image   ',
                                    maxval=filesize)
        digest = hashlib.md5()
        with open(filename) as file_:
            pbar.start()
            while file_.tell() < filesize:
                chunk = file_.read(4096)
                digest.update(chunk)
                pbar.update(file_.tell())
        pbar.finish()
        crc = zlib.crc32(digest.hexdigest()) & 0xffffffff
        return '{0:0>10d}'.format(crc)

    def bundle_and_register_all(self, workdir, tarball_filename):
        if self.args['show_progress']:
            print 'Preparing to extract image...'
        if self.args.get('pad_name', False):
            image_name_pad = '{0:0>8x}-'.format(random.randrange(16**8))
        else:
            image_name_pad = ''
        image_name = 'eustore-{0}{1}'.format(
            image_name_pad,
            os.path.splitext(os.path.basename(tarball_filename))[0]
            .replace('.', '_'))
        tarball = tarfile.open(tarball_filename, 'r:gz')
        try:
            members = tarball.getmembers()
            filenames = tuple(member.name for member in members)
            commonprefix = os.path.commonprefix(filenames)
            kernel_id = self.args.get('kernel')
            ramdisk_id = self.args.get('ramdisk')
            if self.args['hypervisor'] == 'universal':
                hv_prefix = commonprefix
            else:
                hv_type_dir = self.args['hypervisor'] + '-kernel'
                hv_prefix = os.path.join(commonprefix, hv_type_dir)
            # Get any kernel and ramdisk images we're missing
            bundled_images = []
            for member in members:
                if member.name.startswith(hv_prefix):
                    if kernel_id is None and 'vmlinu' in member.name:
                        # Note that vmlinux/vmlinuz is not always at the
                        # beginning of the file name
                        bundled_images.append(member.name)
                        kernel_image = self.extract_without_path(
                            tarball, member, workdir, 'Extracting kernel ')
                        manifest_loc = self.bundle_and_upload_image(
                            kernel_image, 'kernel', workdir)
                        req = RegisterImage(
                            config=self.config, service=self.__eucalyptus,
                            ImageLocation=manifest_loc,
                            Name=(image_name + '-kernel'),
                            Description=self.args.get('description'),
                            Architecture=self.args.get('architecture'))
                        response = req.main()
                        kernel_id = response.get('imageId')
                        if self.args['show_progress']:
                            print 'Registered kernel image', kernel_id
                    elif (ramdisk_id is None and
                          any(s in member.name for s in ('initrd', 'initramfs',
                                                         'loader'))):
                        bundled_images.append(member.name)
                        ramdisk_image = self.extract_without_path(
                            tarball, member, workdir, 'Extracting ramdisk')
                        manifest_loc = self.bundle_and_upload_image(
                            ramdisk_image, 'ramdisk', workdir)
                        req = RegisterImage(
                            config=self.config, service=self.__eucalyptus,
                            ImageLocation=manifest_loc,
                            Name=(image_name + '-ramdisk'),
                            Description=self.args.get('description'),
                            Architecture=self.args.get('architecture'))
                        response = req.main()
                        ramdisk_id = response.get('imageId')
                        if self.args['show_progress']:
                            print 'Registered ramdisk image', ramdisk_id
            if kernel_id is None:
                raise RuntimeError('failed to find a useful kernel image')
            if ramdisk_id is None:
                raise RuntimeError('failed to find a useful ramdisk image')
            # Now that we have kernel and ramdisk image IDs, deal with the
            # machine image
            machine_id = None
            for member in members:
                if member.name in bundled_images:
                    continue
                if any(s in member.name for s in ('initrd', 'initramfs',
                                                  'loader')):
                    # Make sure we don't accidentally register a ramdisk image.
                    # This can happen when use of --ramdisk prevents us from
                    # pruning it later.
                    continue
                if machine_id is None and member.name.endswith('.img'):
                    bundled_images.append(member.name)
                    machine_image = self.extract_without_path(
                        tarball, member, workdir, 'Extracting image  ')
                    manifest_loc = self.bundle_and_upload_image(
                        machine_image, 'machine', workdir, kernel_id=kernel_id,
                        ramdisk_id=ramdisk_id)
                    req = RegisterImage(
                        config=self.config, service=self.__eucalyptus,
                        ImageLocation=manifest_loc, Name=image_name,
                        Description=self.args.get('description'),
                        Architecture=self.args.get('architecture'))
                    response = req.main()
                    machine_id = response.get('imageId')
                    if self.args['show_progress']:
                        print 'Registered machine image', machine_id
        finally:
            tarball.close()
        if self.args['show_progress']:
            print '-- Done --'
        return {'machine': machine_id, 'kernel': kernel_id,
                'ramdisk': ramdisk_id}

    def extract_without_path(self, tarball, member, destdir, bar_label):
        dest_filename = os.path.join(destdir, os.path.basename(member.name))
        self.log.info('extracting %s from tarball to %s', member.name,
                      dest_filename)
        src = tarball.extractfile(member)
        pbar = self.get_progressbar(label=bar_label, maxval=member.size)
        try:
            with open(dest_filename, 'w') as dest:
                while dest.tell() < member.size:
                    # The first chunk may take a while to read since gzip
                    # doesn't support seeking.
                    chunk = src.read(16384)
                    dest.write(chunk)
                    if pbar.start_time is None:
                        pbar.start()
                    pbar.update(dest.tell())
                pbar.finish()
        finally:
            src.close()
        return dest_filename

    def bundle_and_upload_image(self, image, image_type, workdir,
                                kernel_id=None, ramdisk_id=None):
        unique_bucket = self.args['bucket']
        if image_type == 'machine':
            image_type_args = {'kernel': kernel_id,
                               'ramdisk': ramdisk_id}
            progressbar_label = 'Bundling image    '
        elif image_type == 'kernel':
            image_type_args = {'kernel': 'true'}
            progressbar_label = 'Bundling kernel   '
            if self.args.get('separate_buckets'):
                unique_bucket = ''.join([self.args['bucket'], '-kernel'])
        elif image_type == 'ramdisk':
            image_type_args = {'ramdisk': 'true'}
            progressbar_label = 'Bundling ramdisk  '
            if self.args.get('separate_buckets'):
                unique_bucket = ''.join([self.args['bucket'], '-ramdisk'])
        else:
            raise ValueError("unrecognized image type: '{0}'"
                             .format(image_type))
        cmd = BundleImage(config=self.config, image=image,
                          arch=self.args['architecture'],
                          cert=self.args['cert'],
                          privatekey=self.args['privatekey'],
                          ec2cert=self.args['ec2cert'], user=self.args['user'],
                          destination=workdir, image_type=image_type,
                          show_progress=self.args.get('show_progress', False),
                          progressbar_label=progressbar_label,
                          **image_type_args)
        __, manifest_path = cmd.main()

        if self.args.get('show_progress', False):
            print '-- Uploading {0} image --'.format(image_type)
        cmd = UploadBundle(config=self.config, service=self.__walrus,
                           bucket=unique_bucket, manifest=manifest_path,
                           acl='aws-exec-read',
                           show_progress=self.args.get('show_progress', False))
        manifest_loc = cmd.main()
        return manifest_loc
Пример #2
0
    def configure(self):
        EuStoreRequest.configure(self)
        set_userregion(self.config, self.args.get('userregion'))

        if self.args.get('kernel_type'):
            # Use it and complain
            self.args['hypervisor'] = self.args['kernel_type']
            msg = ('argument -k/--kernel-type is deprecated; use --hypervisor '
                   'instead')
            self.log.warn(msg)
            print >> sys.stderr, 'warning:', msg

        # Get bundle creds first
        add_bundle_creds(self.args, self.config)
        if not self.args.get('cert'):
            raise ArgumentError(
                'missing certificate; please supply one with -c')
        self.log.debug('certificate: %s', self.args['cert'])
        if not self.args.get('privatekey'):
            raise ArgumentError(
                'missing private key; please supply one with --privatekey')
        self.log.debug('private key: %s', self.args['privatekey'])
        if not self.args.get('ec2cert'):
            raise ArgumentError(
                'missing cloud certificate; please supply one with --ec2cert')
        self.log.debug('cloud certificate: %s', self.args['ec2cert'])
        if not self.args.get('user'):
            raise ArgumentError(
                'missing account ID; please supply one with --user')
        self.log.debug('account ID: %s', self.args['user'])

        # Set up the web services -- we're going to use them a lot
        query_auth = QuerySigV2Auth(self.config,
                                    key_id=self.args.get('key_id'),
                                    secret_key=self.args.get('secret_key'))
        self.__euare = Euare(self.config, loglevel=self.log.level,
                             auth=copy.copy(query_auth),
                             url=self.args.get('iam_url'))
        self.log.debug('configuring euare service')
        self.__euare.configure()
        self.__eucalyptus = Eucalyptus(self.config, loglevel=self.log.level,
                                       auth=copy.copy(query_auth),
                                       url=self.args.get('ec2_url'))
        self.log.debug('configuring eucalyptus service')
        self.__eucalyptus.configure()
        s3_auth = S3RestAuth(self.config, key_id=self.args.get('key_id'),
                             secret_key=self.args.get('secret_key'))
        self.__walrus = Walrus(self.config, loglevel=self.log.level,
                               auth=s3_auth, url=self.args.get('s3_url'))
        self.__walrus.configure()

        # Check other args next
        if self.args.get('tarball'):
            if not self.args.get('architecture'):
                raise ArgumentError('argument -a/--architecture is required '
                                    'when -t/--tarball is used')
            if not self.args.get('hypervisor'):
                raise ArgumentError('argument --hypervisor is required when '
                                    '-t/--tarball is used')
            self.args['tarball'] = os.path.expanduser(os.path.expandvars(
                self.args['tarball']))
            if not os.path.exists(self.args['tarball']):
                raise ArgumentError("tarball file '{0}' does not exist"
                                    .format(self.args['tarball']))
            if not os.path.isfile(self.args['tarball']):
                raise ArgumentError("tarball file '{0}' is not a file"
                                    .format(self.args['tarball']))
        if self.args.get('kernel') and not self.args.get('ramdisk'):
            raise ArgumentError('argument --kernel: --ramdisk is required')
        if self.args.get('ramdisk') and not self.args.get('kernel'):
            raise ArgumentError('argument --ramdisk: --kernel is required')
        if self.args.get('image') and self.args.get('architecture'):
            self.log.warn("downloaded image's architecture may be overridden")