Example #1
0
    def cache(self, fetch_func, filename, size=None, *args, **kwargs):
        """Creates image from template.

        Ensures that template and image not already exists.
        Ensures that base directory exists.
        Synchronizes on template fetching.

        :fetch_func: Function that creates the base image
                     Should accept `target` argument.
        :filename: Name of the file in the image directory
        :size: Size of created image in bytes (optional)
        """
        @utils.synchronized(filename, external=True, lock_path=self.lock_path)
        def fetch_func_sync(target, *args, **kwargs):
            # The image may have been fetched while a subsequent
            # call was waiting to obtain the lock.
            if not os.path.exists(target):
                fetch_func(target=target, *args, **kwargs)

        base_dir = os.path.join(CONF.instances_path,
                                CONF.image_cache_subdirectory_name)
        if not os.path.exists(base_dir):
            fileutils.ensure_tree(base_dir)
        base = os.path.join(base_dir, filename)

        if not self.check_image_exists() or not os.path.exists(base):
            self.create_image(fetch_func_sync, base, size,
                              *args, **kwargs)

        if (size and self.preallocate and self._can_fallocate() and
                os.access(self.path, os.W_OK)):
            utils.execute('fallocate', '-n', '-l', size, self.path)
Example #2
0
def write_stored_info(target, field=None, value=None):
    """Write information about an image."""

    if not field:
        return

    info_file = get_info_filename(target)
    LOG.info(_LI('Writing stored info to %s'), info_file)
    fileutils.ensure_tree(os.path.dirname(info_file))

    lock_name = 'info-%s' % os.path.split(target)[-1]
    lock_path = os.path.join(CONF.instances_path, 'locks')

    @utils.synchronized(lock_name, external=True, lock_path=lock_path)
    def write_file(info_file, field, value):
        d = {}

        if os.path.exists(info_file):
            with open(info_file, 'r') as f:
                d = _read_possible_json(f.read(), info_file)

        d[field] = value
        d['%s-timestamp' % field] = time.time()

        with open(info_file, 'w') as f:
            f.write(jsonutils.dumps(d))

    write_file(info_file, field, value)
Example #3
0
def write_stored_info(target, field=None, value=None):
    """Write information about an image."""

    if not field:
        return

    info_file = get_info_filename(target)
    LOG.info(_LI('Writing stored info to %s'), info_file)
    fileutils.ensure_tree(os.path.dirname(info_file))

    lock_name = 'info-%s' % os.path.split(target)[-1]
    lock_path = os.path.join(CONF.instances_path, 'locks')

    @utils.synchronized(lock_name, external=True, lock_path=lock_path)
    def write_file(info_file, field, value):
        d = {}

        if os.path.exists(info_file):
            with open(info_file, 'r') as f:
                d = _read_possible_json(f.read(), info_file)

        d[field] = value
        d['%s-timestamp' % field] = time.time()

        with open(info_file, 'w') as f:
            f.write(jsonutils.dumps(d))

    write_file(info_file, field, value)
Example #4
0
    def cache(self, fetch_func, filename, size=None, *args, **kwargs):
        """Creates image from template.

        Ensures that template and image not already exists.
        Ensures that base directory exists.
        Synchronizes on template fetching.

        :fetch_func: Function that creates the base image
                     Should accept `target` argument.
        :filename: Name of the file in the image directory
        :size: Size of created image in bytes (optional)
        """
        @utils.synchronized(filename, external=True, lock_path=self.lock_path)
        def fetch_func_sync(target, *args, **kwargs):
            # The image may have been fetched while a subsequent
            # call was waiting to obtain the lock.
            if not os.path.exists(target):
                fetch_func(target=target, *args, **kwargs)

        base_dir = os.path.join(CONF.instances_path,
                                CONF.image_cache_subdirectory_name)
        if not os.path.exists(base_dir):
            fileutils.ensure_tree(base_dir)
        base = os.path.join(base_dir, filename)

        if not self.check_image_exists() or not os.path.exists(base):
            self.create_image(fetch_func_sync, base, size, *args, **kwargs)

        if (size and self.preallocate and self._can_fallocate()
                and os.access(self.path, os.W_OK)):
            utils.execute('fallocate', '-n', '-l', size, self.path)
Example #5
0
def _sign_csr(csr_text, ca_folder):
    with utils.tempdir() as tmpdir:
        inbound = os.path.join(tmpdir, 'inbound.csr')
        outbound = os.path.join(tmpdir, 'outbound.csr')

        try:
            with open(inbound, 'w') as csrfile:
                csrfile.write(csr_text)
        except IOError:
            with excutils.save_and_reraise_exception():
                LOG.exception(_LE('Failed to write inbound.csr'))

        LOG.debug('Flags path: %s', ca_folder)
        start = os.getcwd()

        # Change working dir to CA
        fileutils.ensure_tree(ca_folder)
        os.chdir(ca_folder)
        utils.execute('openssl', 'ca', '-batch', '-out', outbound, '-config',
                      './openssl.cnf', '-infiles', inbound)
        out, _err = utils.execute('openssl', 'x509', '-in', outbound,
                                  '-serial', '-noout')
        serial = string.strip(out.rpartition('=')[2])
        os.chdir(start)

        with open(outbound, 'r') as crtfile:
            return (serial, crtfile.read())
Example #6
0
def _sign_csr(csr_text, ca_folder):
    with utils.tempdir() as tmpdir:
        inbound = os.path.join(tmpdir, 'inbound.csr')
        outbound = os.path.join(tmpdir, 'outbound.csr')

        try:
            with open(inbound, 'w') as csrfile:
                csrfile.write(csr_text)
        except IOError:
            with excutils.save_and_reraise_exception():
                LOG.exception(_LE('Failed to write inbound.csr'))

        LOG.debug('Flags path: %s', ca_folder)
        start = os.getcwd()

        # Change working dir to CA
        fileutils.ensure_tree(ca_folder)
        os.chdir(ca_folder)
        utils.execute('openssl', 'ca', '-batch', '-out', outbound, '-config',
                      './openssl.cnf', '-infiles', inbound)
        out, _err = utils.execute('openssl', 'x509', '-in', outbound,
                                  '-serial', '-noout')
        serial = string.strip(out.rpartition('=')[2])
        os.chdir(start)

        with open(outbound, 'r') as crtfile:
            return (serial, crtfile.read())
Example #7
0
    def resolve_driver_format(self):
        """Return the driver format for self.path.

        First checks self.disk_info_path for an entry.
        If it's not there, calls self._get_driver_format(), and then
        stores the result in self.disk_info_path

        See https://bugs.launchpad.net/patron/+bug/1221190
        """
        def _dict_from_line(line):
            if not line:
                return {}
            try:
                return jsonutils.loads(line)
            except (TypeError, ValueError) as e:
                msg = (_("Could not load line %(line)s, got error "
                         "%(error)s") % {
                             'line': line,
                             'error': e
                         })
                raise exception.InvalidDiskInfo(reason=msg)

        @utils.synchronized(self.disk_info_path,
                            external=False,
                            lock_path=self.lock_path)
        def write_to_disk_info_file():
            # Use os.open to create it without group or world write permission.
            fd = os.open(self.disk_info_path, os.O_RDONLY | os.O_CREAT, 0o644)
            with os.fdopen(fd, "r") as disk_info_file:
                line = disk_info_file.read().rstrip()
                dct = _dict_from_line(line)

            if self.path in dct:
                msg = _("Attempted overwrite of an existing value.")
                raise exception.InvalidDiskInfo(reason=msg)
            dct.update({self.path: driver_format})

            tmp_path = self.disk_info_path + ".tmp"
            fd = os.open(tmp_path, os.O_WRONLY | os.O_CREAT, 0o644)
            with os.fdopen(fd, "w") as tmp_file:
                tmp_file.write('%s\n' % jsonutils.dumps(dct))
            os.rename(tmp_path, self.disk_info_path)

        try:
            if (self.disk_info_path is not None
                    and os.path.exists(self.disk_info_path)):
                with open(self.disk_info_path) as disk_info_file:
                    line = disk_info_file.read().rstrip()
                    dct = _dict_from_line(line)
                    for path, driver_format in dct.iteritems():
                        if path == self.path:
                            return driver_format
            driver_format = self._get_driver_format()
            if self.disk_info_path is not None:
                fileutils.ensure_tree(os.path.dirname(self.disk_info_path))
                write_to_disk_info_file()
        except OSError as e:
            raise exception.DiskInfoReadWriteFail(reason=six.text_type(e))
        return driver_format
Example #8
0
 def put(self, bucket_name):
     path = os.path.abspath(
         os.path.join(self.application.directory, bucket_name))
     if (not path.startswith(self.application.directory)
             or os.path.exists(path)):
         self.set_status(403)
         return
     fileutils.ensure_tree(path)
     self.finish()
Example #9
0
 def put(self, bucket_name):
     path = os.path.abspath(os.path.join(
         self.application.directory, bucket_name))
     if (not path.startswith(self.application.directory) or
             os.path.exists(path)):
         self.set_status(403)
         return
     fileutils.ensure_tree(path)
     self.finish()
Example #10
0
    def resolve_driver_format(self):
        """Return the driver format for self.path.

        First checks self.disk_info_path for an entry.
        If it's not there, calls self._get_driver_format(), and then
        stores the result in self.disk_info_path

        See https://bugs.launchpad.net/patron/+bug/1221190
        """
        def _dict_from_line(line):
            if not line:
                return {}
            try:
                return jsonutils.loads(line)
            except (TypeError, ValueError) as e:
                msg = (_("Could not load line %(line)s, got error "
                        "%(error)s") %
                        {'line': line, 'error': e})
                raise exception.InvalidDiskInfo(reason=msg)

        @utils.synchronized(self.disk_info_path, external=False,
                            lock_path=self.lock_path)
        def write_to_disk_info_file():
            # Use os.open to create it without group or world write permission.
            fd = os.open(self.disk_info_path, os.O_RDONLY | os.O_CREAT, 0o644)
            with os.fdopen(fd, "r") as disk_info_file:
                line = disk_info_file.read().rstrip()
                dct = _dict_from_line(line)

            if self.path in dct:
                msg = _("Attempted overwrite of an existing value.")
                raise exception.InvalidDiskInfo(reason=msg)
            dct.update({self.path: driver_format})

            tmp_path = self.disk_info_path + ".tmp"
            fd = os.open(tmp_path, os.O_WRONLY | os.O_CREAT, 0o644)
            with os.fdopen(fd, "w") as tmp_file:
                tmp_file.write('%s\n' % jsonutils.dumps(dct))
            os.rename(tmp_path, self.disk_info_path)

        try:
            if (self.disk_info_path is not None and
                        os.path.exists(self.disk_info_path)):
                with open(self.disk_info_path) as disk_info_file:
                    line = disk_info_file.read().rstrip()
                    dct = _dict_from_line(line)
                    for path, driver_format in dct.iteritems():
                        if path == self.path:
                            return driver_format
            driver_format = self._get_driver_format()
            if self.disk_info_path is not None:
                fileutils.ensure_tree(os.path.dirname(self.disk_info_path))
                write_to_disk_info_file()
        except OSError as e:
            raise exception.DiskInfoReadWriteFail(reason=six.text_type(e))
        return driver_format
Example #11
0
def mount_volume(volume, mnt_base, configfile=None):
    """Wraps execute calls for mounting a Quobyte volume"""
    fileutils.ensure_tree(mnt_base)

    command = ['mount.quobyte', volume, mnt_base]
    if configfile:
        command.extend(['-c', configfile])

    LOG.debug('Mounting volume %s at mount point %s ...', volume, mnt_base)
    # Run mount command but do not fail on already mounted exit code
    utils.execute(*command, check_exit_code=[0, 4])
    LOG.info(_LI('Mounted volume: %s'), volume)
Example #12
0
def mount_volume(volume, mnt_base, configfile=None):
    """Wraps execute calls for mounting a Quobyte volume"""
    fileutils.ensure_tree(mnt_base)

    command = ['mount.quobyte', volume, mnt_base]
    if configfile:
        command.extend(['-c', configfile])

    LOG.debug('Mounting volume %s at mount point %s ...',
              volume,
              mnt_base)
    # Run mount command but do not fail on already mounted exit code
    utils.execute(*command, check_exit_code=[0, 4])
    LOG.info(_LI('Mounted volume: %s'), volume)
Example #13
0
 def setup_key_pair(self, context):
     key_name = '%s%s' % (context.project_id, CONF.vpn_key_suffix)
     try:
         keypair_api = compute.api.KeypairAPI()
         result, private_key = keypair_api.create_key_pair(
             context, context.user_id, key_name)
         key_dir = os.path.join(CONF.keys_path, context.user_id)
         fileutils.ensure_tree(key_dir)
         key_path = os.path.join(key_dir, '%s.pem' % key_name)
         with open(key_path, 'w') as f:
             f.write(private_key)
     except (exception.KeyPairExists, os.error, IOError):
         pass
     return key_name
Example #14
0
    def create_image(self, prepare_template, base, size, *args, **kwargs):
        filename = os.path.split(base)[-1]

        @utils.synchronized(filename, external=True, lock_path=self.lock_path)
        def create_ploop_image(base, target, size):
            image_path = os.path.join(target, "root.hds")
            libvirt_utils.copy_image(base, image_path)
            utils.execute('ploop', 'restore-descriptor', '-f', self.pcs_format,
                          target, image_path)
            if size:
                dd_path = os.path.join(self.path, "DiskDescriptor.xml")
                utils.execute('ploop',
                              'grow',
                              '-s',
                              '%dK' % (size >> 10),
                              dd_path,
                              run_as_root=True)

        if not os.path.exists(self.path):
            if CONF.force_raw_images:
                self.pcs_format = "raw"
            else:
                image_meta = IMAGE_API.get(kwargs["context"],
                                           kwargs["image_id"])
                format = image_meta.get("disk_format")
                if format == "ploop":
                    self.pcs_format = "expanded"
                elif format == "raw":
                    self.pcs_format = "raw"
                else:
                    reason = _("PCS doesn't support images in %s format."
                               " You should either set force_raw_images=True"
                               " in config or upload an image in ploop"
                               " or raw format.") % format
                    raise exception.ImageUnacceptable(
                        image_id=kwargs["image_id"], reason=reason)

        if not os.path.exists(base):
            prepare_template(target=base, max_size=size, *args, **kwargs)
        self.verify_base_size(base, size)

        if os.path.exists(self.path):
            return

        fileutils.ensure_tree(self.path)

        remove_func = functools.partial(fileutils.delete_if_exists,
                                        remove=shutil.rmtree)
        with fileutils.remove_path_on_error(self.path, remove=remove_func):
            create_ploop_image(base, self.path, size)
Example #15
0
def ensure_ca_filesystem():
    """Ensure the CA filesystem exists."""
    ca_dir = ca_folder()
    if not os.path.exists(ca_path()):
        genrootca_sh_path = os.path.abspath(
                os.path.join(os.path.dirname(__file__), 'CA', 'genrootca.sh'))

        start = os.getcwd()
        fileutils.ensure_tree(ca_dir)
        os.chdir(ca_dir)
        try:
            utils.execute("sh", genrootca_sh_path)
        finally:
            os.chdir(start)
Example #16
0
def ensure_ca_filesystem():
    """Ensure the CA filesystem exists."""
    ca_dir = ca_folder()
    if not os.path.exists(ca_path()):
        genrootca_sh_path = os.path.abspath(
            os.path.join(os.path.dirname(__file__), 'CA', 'genrootca.sh'))

        start = os.getcwd()
        fileutils.ensure_tree(ca_dir)
        os.chdir(ca_dir)
        try:
            utils.execute("sh", genrootca_sh_path)
        finally:
            os.chdir(start)
Example #17
0
 def setup_key_pair(self, context):
     key_name = '%s%s' % (context.project_id, CONF.vpn_key_suffix)
     try:
         keypair_api = compute.api.KeypairAPI()
         result, private_key = keypair_api.create_key_pair(context,
                                                           context.user_id,
                                                           key_name)
         key_dir = os.path.join(CONF.keys_path, context.user_id)
         fileutils.ensure_tree(key_dir)
         key_path = os.path.join(key_dir, '%s.pem' % key_name)
         with open(key_path, 'w') as f:
             f.write(private_key)
     except (exception.KeyPairExists, os.error, IOError):
         pass
     return key_name
Example #18
0
    def create_image(self, prepare_template, base, size, *args, **kwargs):
        filename = os.path.split(base)[-1]

        @utils.synchronized(filename, external=True, lock_path=self.lock_path)
        def create_ploop_image(base, target, size):
            image_path = os.path.join(target, "root.hds")
            libvirt_utils.copy_image(base, image_path)
            utils.execute('ploop', 'restore-descriptor', '-f', self.pcs_format,
                          target, image_path)
            if size:
                dd_path = os.path.join(self.path, "DiskDescriptor.xml")
                utils.execute('ploop', 'grow', '-s', '%dK' % (size >> 10),
                              dd_path, run_as_root=True)

        if not os.path.exists(self.path):
            if CONF.force_raw_images:
                self.pcs_format = "raw"
            else:
                image_meta = IMAGE_API.get(kwargs["context"],
                                           kwargs["image_id"])
                format = image_meta.get("disk_format")
                if format == "ploop":
                    self.pcs_format = "expanded"
                elif format == "raw":
                    self.pcs_format = "raw"
                else:
                    reason = _("PCS doesn't support images in %s format."
                                " You should either set force_raw_images=True"
                                " in config or upload an image in ploop"
                                " or raw format.") % format
                    raise exception.ImageUnacceptable(
                                        image_id=kwargs["image_id"],
                                        reason=reason)

        if not os.path.exists(base):
            prepare_template(target=base, max_size=size, *args, **kwargs)
        self.verify_base_size(base, size)

        if os.path.exists(self.path):
            return

        fileutils.ensure_tree(self.path)

        remove_func = functools.partial(fileutils.delete_if_exists,
                                        remove=shutil.rmtree)
        with fileutils.remove_path_on_error(self.path, remove=remove_func):
            create_ploop_image(base, self.path, size)
Example #19
0
 def put(self, bucket, object_name):
     object_name = urllib.unquote(object_name)
     bucket_dir = os.path.abspath(
         os.path.join(self.application.directory, bucket))
     if (not bucket_dir.startswith(self.application.directory)
             or not os.path.isdir(bucket_dir)):
         self.set_404()
         return
     path = self._object_path(bucket, object_name)
     if not path.startswith(bucket_dir) or os.path.isdir(path):
         self.set_status(403)
         return
     directory = os.path.dirname(path)
     fileutils.ensure_tree(directory)
     object_file = open(path, "w")
     object_file.write(self.request.body)
     object_file.close()
     self.set_header('ETag', '"%s"' % utils.get_hash_str(self.request.body))
     self.finish()
Example #20
0
 def put(self, bucket, object_name):
     object_name = urllib.unquote(object_name)
     bucket_dir = os.path.abspath(os.path.join(
         self.application.directory, bucket))
     if (not bucket_dir.startswith(self.application.directory) or
             not os.path.isdir(bucket_dir)):
         self.set_404()
         return
     path = self._object_path(bucket, object_name)
     if not path.startswith(bucket_dir) or os.path.isdir(path):
         self.set_status(403)
         return
     directory = os.path.dirname(path)
     fileutils.ensure_tree(directory)
     object_file = open(path, "w")
     object_file.write(self.request.body)
     object_file.close()
     self.set_header('ETag',
                     '"%s"' % utils.get_hash_str(self.request.body))
     self.finish()
Example #21
0
    def __init__(self, root_directory, bucket_depth=0, mapper=None):
        versionutils.report_deprecated_feature(
            LOG,
            _LW('The in tree EC2 API is deprecated as of Kilo release and may '
                'be removed in a future release. The stackforge ec2-api '
                'project http://git.openstack.org/cgit/stackforge/ec2-api/ '
                'is the target replacement for this functionality.')
        )
        if mapper is None:
            mapper = routes.Mapper()

        mapper.connect('/',
                controller=lambda *a, **kw: RootHandler(self)(*a, **kw))
        mapper.connect('/{bucket}/{object_name}',
                controller=lambda *a, **kw: ObjectHandler(self)(*a, **kw))
        mapper.connect('/{bucket_name}/',
                controller=lambda *a, **kw: BucketHandler(self)(*a, **kw))
        self.directory = os.path.abspath(root_directory)
        fileutils.ensure_tree(self.directory)
        self.bucket_depth = bucket_depth
        super(S3Application, self).__init__(mapper)
Example #22
0
    def __init__(self, root_directory, bucket_depth=0, mapper=None):
        versionutils.report_deprecated_feature(
            LOG,
            _LW('The in tree EC2 API is deprecated as of Kilo release and may '
                'be removed in a future release. The stackforge ec2-api '
                'project http://git.openstack.org/cgit/stackforge/ec2-api/ '
                'is the target replacement for this functionality.'))
        if mapper is None:
            mapper = routes.Mapper()

        mapper.connect('/',
                       controller=lambda *a, **kw: RootHandler(self)(*a, **kw))
        mapper.connect('/{bucket}/{object_name}',
                       controller=lambda *a, **kw: ObjectHandler(self)
                       (*a, **kw))
        mapper.connect('/{bucket_name}/',
                       controller=lambda *a, **kw: BucketHandler(self)
                       (*a, **kw))
        self.directory = os.path.abspath(root_directory)
        fileutils.ensure_tree(self.directory)
        self.bucket_depth = bucket_depth
        super(S3Application, self).__init__(mapper)
Example #23
0
 def _add_file(self, basedir, path, data):
     filepath = os.path.join(basedir, path)
     dirname = os.path.dirname(filepath)
     fileutils.ensure_tree(dirname)
     with open(filepath, 'wb') as f:
         f.write(data)
Example #24
0
 def _add_file(self, basedir, path, data):
     filepath = os.path.join(basedir, path)
     dirname = os.path.dirname(filepath)
     fileutils.ensure_tree(dirname)
     with open(filepath, 'wb') as f:
         f.write(data)
Example #25
0
 def setup(self):
     """Ensure the keychains and folders exist."""
     # NOTE(vish): One of the drawbacks of doing this in the api is
     #             the keys will only be on the api node that launched
     #             the cloudpipe.
     fileutils.ensure_tree(CONF.keys_path)