def _check_for_path_errors(self, pointer):
     if not re.match(self.PATH_REGEX_COMPILED, pointer):
         msg = _("Json path should start with a '/', " "end with no '/', no 2 subsequent '/' are allowed.")
         raise exc.InvalidJsonPatchPath(path=pointer, explanation=msg)
     if re.search("~[^01]", pointer) or pointer.endswith("~"):
         msg = _("Pointer contains '~' which is not part of" " a recognized escape sequence [~0, ~1].")
         raise exc.InvalidJsonPatchPath(path=pointer, explanation=msg)
Example #2
0
    def launch(pid_file, conf_file=None, capture_output=False, await_time=0):
        args = [server]
        if conf_file:
            args += ['--config-file', conf_file]
            msg = (_('%(verb)sing %(serv)s with %(conf)s') %
                   {'verb': verb, 'serv': server, 'conf': conf_file})
        else:
            msg = (_('%(verb)sing %(serv)s') % {'verb': verb, 'serv': server})
        print(msg)

        close_stdio_on_exec()

        pid = os.fork()
        if pid == 0:
            os.setsid()
            redirect_stdio(server, capture_output)
            try:
                os.execlp('%s' % server, *args)
            except OSError as e:
                msg = (_('unable to launch %(serv)s. Got error: %(e)s') %
                       {'serv': server, 'e': e})
                sys.exit(msg)
            sys.exit(0)
        else:
            write_pid_file(pid_file, pid)
            await_child(pid, await_time)
            return pid
Example #3
0
def do_stop(server, args, graceful=False):
    if graceful and server in GRACEFUL_SHUTDOWN_SERVERS:
        sig = signal.SIGHUP
    else:
        sig = signal.SIGTERM

    did_anything = False
    pfiles = pid_files(server, CONF.pid_file)
    for pid_file, pid in pfiles:
        did_anything = True
        try:
            os.unlink(pid_file)
        except OSError:
            pass
        try:
            print(_('Stopping %(serv)s (pid %(pid)s) with signal(%(sig)s)')
                  % {'serv': server, 'pid': pid, 'sig': sig})
            os.kill(pid, sig)
        except OSError:
            print(_("Process %d not running") % pid)
    for pid_file, pid in pfiles:
        for _junk in range(150):  # 15 seconds
            if not os.path.exists('/proc/%s' % pid):
                break
            time.sleep(0.1)
        else:
            print(_('Waited 15 seconds for pid %(pid)s (%(file)s) to die;'
                    ' giving up') % {'pid': pid, 'file': pid_file})
    if not did_anything:
        print(_('%s is already stopped') % server)
Example #4
0
def do_check_status(pid_file, server):
    if os.path.exists(pid_file):
        with open(pid_file, 'r') as pidfile:
            pid = pidfile.read().strip()
        print(_("%(serv)s (pid %(pid)s) is running...") %
              {'serv': server, 'pid': pid})
    else:
        print(_("%s is stopped") % server)
Example #5
0
    def _validate_actions(self, actions):
        if not actions:
            msg = _("actions param cannot be empty")
            raise webob.exc.HTTPBadRequest(explanation=msg)

        output = []
        allowed_action_types = ['create', 'update', 'delete', 'index']
        for action in actions:
            action_type = action.get('action', 'index')
            document_id = action.get('id')
            document_type = action.get('type')
            index_name = action.get('index')
            data = action.get('data', {})
            script = action.get('script')

            if index_name is not None:
                index_name = self._validate_index(index_name)

            if document_type is not None:
                document_type = self._validate_doc_type(document_type)

            if action_type not in allowed_action_types:
                msg = _("Invalid action type: '%s'") % action_type
                raise webob.exc.HTTPBadRequest(explanation=msg)
            elif (action_type in ['create', 'update', 'index'] and
                    not any([data, script])):
                msg = (_("Action type '%s' requires data or script param.") %
                       action_type)
                raise webob.exc.HTTPBadRequest(explanation=msg)
            elif action_type in ['update', 'delete'] and not document_id:
                msg = (_("Action type '%s' requires ID of the document.") %
                       action_type)
                raise webob.exc.HTTPBadRequest(explanation=msg)

            bulk_action = {
                '_op_type': action_type,
                '_id': document_id,
                '_index': index_name,
                '_type': document_type,
            }

            if script:
                data_field = 'params'
                bulk_action['script'] = script
            elif action_type == 'update':
                data_field = 'doc'
            else:
                data_field = '_source'

            bulk_action[data_field] = data

            output.append(bulk_action)
        return output
Example #6
0
    def _validate_limit(self, limit):
        try:
            limit = int(limit)
        except ValueError:
            msg = _("limit param must be an integer")
            raise webob.exc.HTTPBadRequest(explanation=msg)

        if limit < 0:
            msg = _("limit param must be a non-negative integer")
            raise webob.exc.HTTPBadRequest(explanation=msg)

        return limit
Example #7
0
    def _validate_offset(self, offset):
        try:
            offset = int(offset)
        except ValueError:
            msg = _("offset param must be an integer")
            raise webob.exc.HTTPBadRequest(explanation=msg)

        if offset < 0:
            msg = _("offset param must be positive")
            raise webob.exc.HTTPBadRequest(explanation=msg)

        return offset
Example #8
0
    def _validate_integer_param(self, value, gte, param_name):
        try:
            value = int(value)
        except ValueError:
            msg = _("%s param must be an integer") % param_name
            raise webob.exc.HTTPBadRequest(explanation=msg)

        if value < gte:
            msg = _("%(param_name)s param must be greater than or equal "
                    "to %(gte)s") % {'param_name': param_name, 'gte': gte}
            raise webob.exc.HTTPBadRequest(explanation=msg)

        return value
Example #9
0
 def _check_dict(data_dict):
     # a dict of dicts has to be checked recursively
     for key, value in data_dict.items():
         if isinstance(value, dict):
             _check_dict(value)
         else:
             if _is_match(key):
                 msg = _("Property names can't contain 4 byte unicode.")
                 raise exception.Invalid(msg)
             if _is_match(value):
                 msg = (_("%s can't contain 4 byte unicode characters.")
                        % key.title())
                 raise exception.Invalid(msg)
Example #10
0
def validate_key_cert(key_file, cert_file):
    try:
        error_key_name = "private key"
        error_filename = key_file
        with open(key_file, 'r') as keyfile:
            key_str = keyfile.read()
        key = crypto.load_privatekey(crypto.FILETYPE_PEM, key_str)

        error_key_name = "certificate"
        error_filename = cert_file
        with open(cert_file, 'r') as certfile:
            cert_str = certfile.read()
        cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_str)
    except IOError as ioe:
        raise RuntimeError(_("There is a problem with your %(error_key_name)s "
                             "%(error_filename)s.  Please verify it."
                             "  Error: %(ioe)s") %
                           {'error_key_name': error_key_name,
                            'error_filename': error_filename,
                            'ioe': ioe})
    except crypto.Error as ce:
        raise RuntimeError(_("There is a problem with your %(error_key_name)s "
                             "%(error_filename)s.  Please verify it. OpenSSL"
                             " error: %(ce)s") %
                           {'error_key_name': error_key_name,
                            'error_filename': error_filename,
                            'ce': ce})

    try:
        data = uuidutils.generate_uuid()
        digest = CONF.digest_algorithm
        if digest == 'sha1':
            LOG.warning(
                ('The FIPS (FEDERAL INFORMATION PROCESSING STANDARDS)'
                 ' state that the SHA-1 is not suitable for'
                 ' general-purpose digital signature applications (as'
                 ' specified in FIPS 186-3) that require 112 bits of'
                 ' security. The default value is sha1 in Kilo for a'
                 ' smooth upgrade process, and it will be updated'
                 ' with sha256 in next release(L).'))
        out = crypto.sign(key, data, digest)
        crypto.verify(cert, out, data, digest)
    except crypto.Error as ce:
        raise RuntimeError(_("There is a problem with your key pair.  "
                             "Please verify that cert %(cert_file)s and "
                             "key %(key_file)s belong together.  OpenSSL "
                             "error %(ce)s") % {'cert_file': cert_file,
                                                'key_file': key_file,
                                                'ce': ce})
Example #11
0
def get_socket(default_port):
    """
    Bind socket to bind ip:port in conf

    note: Mostly comes from Swift with a few small changes...

    :param default_port: port to bind to if none is specified in conf

    :returns : a socket object as returned from socket.listen or
               ssl.wrap_socket if conf specifies cert_file
    """
    bind_addr = get_bind_addr(default_port)

    # TODO(jaypipes): eventlet's greened socket module does not actually
    # support IPv6 in getaddrinfo(). We need to get around this in the
    # future or monitor upstream for a fix
    address_family = [
        addr[0] for addr in socket.getaddrinfo(bind_addr[0],
                                               bind_addr[1],
                                               socket.AF_UNSPEC,
                                               socket.SOCK_STREAM)
        if addr[0] in (socket.AF_INET, socket.AF_INET6)
    ][0]

    use_ssl = CONF.api.key_file or CONF.api.cert_file
    if use_ssl and (not CONF.api.key_file or not CONF.api.cert_file):
        raise RuntimeError(_("When running server in SSL mode, you must "
                             "specify both a cert_file and key_file "
                             "option value in your configuration file"))

    sock = utils.get_test_suite_socket()
    retry_until = time.time() + 30

    while not sock and time.time() < retry_until:
        try:
            sock = eventlet.listen(bind_addr,
                                   backlog=CONF.api.backlog,
                                   family=address_family)
        except socket.error as err:
            if err.args[0] != errno.EADDRINUSE:
                raise
            eventlet.sleep(0.1)
    if not sock:
        raise RuntimeError(_("Could not bind to %(host)s:%(port)s after"
                             " trying for 30 seconds") %
                           {'host': bind_addr[0],
                            'port': bind_addr[1]})

    return sock
Example #12
0
    def _get_sort_order(self, sort_order):
        if isinstance(sort_order, (six.text_type, dict)):
            # Elasticsearch expects a list
            sort_order = [sort_order]
        elif not isinstance(sort_order, list):
            msg = _("'sort' must be a string, dict or list")
            raise webob.exc.HTTPBadRequest(explanation=msg)

        def replace_sort_field(sort_field):
            # Make some alterations for fields that have a 'raw' field so
            # that documents aren't sorted by tokenized values
            if isinstance(sort_field, six.text_type):
                # Raw field name
                if sort_field in searchlight.elasticsearch.RAW_SORT_FIELDS:
                    return sort_field + ".raw"
            elif isinstance(sort_field, dict):
                for field_name, sort_params in sort_field.items():
                    if field_name in searchlight.elasticsearch.RAW_SORT_FIELDS:
                        # There should only be one object
                        return {field_name + ".raw": sort_params}
            else:
                msg = "Unhandled sort type replacing '%s'" % sort_field
                raise webob.exc.HTTPInternalServerError(explanation=msg)
            return sort_field

        return [replace_sort_field(f) for f in sort_order]
Example #13
0
def get_pid_file(server, pid_file):
    pid_file = (os.path.abspath(pid_file) if pid_file else
                '/var/run/searchlight/%s.pid' % server)
    dir, file = os.path.split(pid_file)

    if not os.path.exists(dir):
        try:
            os.makedirs(dir)
        except OSError:
            pass

    if not os.access(dir, os.W_OK):
        fallback = os.path.join(tempfile.mkdtemp(), '%s.pid' % server)
        msg = (_('Unable to create pid file %(pid)s.  Running as non-root?\n'
                 'Falling back to a temp file, you can stop %(service)s '
                 'service using:\n'
                 '  %(file)s %(server)s stop --pid-file %(fb)s') %
               {'pid': pid_file,
                'service': server,
                'file': __file__,
                'server': server,
                'fb': fallback})
        print(msg)
        pid_file = fallback

    return pid_file
Example #14
0
    def wrapper(*args, **kwargs):

        def _is_match(some_str):
            return (isinstance(some_str, unicode) and
                    REGEX_4BYTE_UNICODE.findall(some_str) != [])

        def _check_dict(data_dict):
            # a dict of dicts has to be checked recursively
            for key, value in data_dict.items():
                if isinstance(value, dict):
                    _check_dict(value)
                else:
                    if _is_match(key):
                        msg = _("Property names can't contain 4 byte unicode.")
                        raise exception.Invalid(msg)
                    if _is_match(value):
                        msg = (_("%s can't contain 4 byte unicode characters.")
                               % key.title())
                        raise exception.Invalid(msg)

        for data_dict in [arg for arg in args if isinstance(arg, dict)]:
            _check_dict(data_dict)
        # now check args for str values
        for arg in args:
            if _is_match(arg):
                msg = _("Param values can't contain 4 byte unicode.")
                raise exception.Invalid(msg)
        # check kwargs as well, as params are passed as kwargs via
        # registry calls
        _check_dict(kwargs)
        return f(*args, **kwargs)
Example #15
0
    def _validate_doc_type(self, doc_type):
        available_types = self._get_available_types()

        if doc_type not in available_types:
            msg = _("Document type '%s' is not supported.") % doc_type
            raise webob.exc.HTTPBadRequest(explanation=msg)

        return doc_type
Example #16
0
    def _validate_index(self, index):
        available_indices = self._get_available_indices()

        if index not in available_indices:
            msg = _("Index '%s' is not supported.") % index
            raise webob.exc.HTTPBadRequest(explanation=msg)

        return index
Example #17
0
def get_content_range(self):
        """Return the `Range` in a request."""
        range_str = self.headers.get('Content-Range')
        if range_str is not None:
            range_ = webob.byterange.ContentRange.parse(range_str)
            if range_ is None:
                msg = _('Malformed Content-Range header: %s') % range_str
                raise webob.exc.HTTPBadRequest(explanation=msg)
            return range_
Example #18
0
def do_reload(pid_file, server):
    if server not in RELOAD_SERVERS:
        msg = (_('Reload of %(serv)s not supported') % {'serv': server})
        sys.exit(msg)

    pid = None
    if os.path.exists(pid_file):
        with open(pid_file, 'r') as pidfile:
            pid = int(pidfile.read().strip())
    else:
        msg = (_('Server %(serv)s is stopped') % {'serv': server})
        sys.exit(msg)

    sig = signal.SIGHUP
    try:
        print(_('Reloading %(serv)s (pid %(pid)s) with signal(%(sig)s)')
              % {'serv': server, 'pid': pid, 'sig': sig})
        os.kill(pid, sig)
    except OSError:
        print(_("Process %d not running") % pid)
Example #19
0
def set_eventlet_hub():
    try:
        eventlet.hubs.use_hub('poll')
    except Exception:
        try:
            eventlet.hubs.use_hub('selects')
        except Exception:
            msg = _("eventlet 'poll' nor 'selects' hubs are available "
                    "on this platform")
            raise exception.WorkerCreationFailure(
                reason=msg)
Example #20
0
def load_paste_app(app_name, flavor=None):
    """
    Builds and returns a WSGI app from a paste config file.

    We assume the last config file specified in the supplied ConfigOpts
    object is the paste config file, if conf_file is None.

    :param app_name: name of the application to load
    :param flavor: name of the variant of the application to load

    :raises RuntimeError when config file cannot be located or application
            cannot be loaded from config file
    """
    # append the deployment flavor to the application name,
    # in order to identify the appropriate paste pipeline
    app_name += _get_deployment_flavor(flavor)

    conf_file = _get_deployment_config_file()
    if conf_file is None:
        raise RuntimeError(_("Unable to locate config file"))

    try:
        logger = logging.getLogger(__name__)
        logger.debug("Loading %(app_name)s from %(conf_file)s",
                     {'conf_file': conf_file, 'app_name': app_name})

        app = deploy.loadapp("config:%s" % conf_file, name=app_name)

        # Log the options used when starting if we're in debug mode...
        if CONF.debug:
            CONF.log_opt_values(logger, logging.DEBUG)

        return app
    except (LookupError, ImportError) as e:
        msg = (_("Unable to load %(app_name)s from "
                 "configuration file %(conf_file)s."
                 "\nGot: %(e)r") % {'app_name': app_name,
                                    'conf_file': conf_file,
                                    'e': e})
        logger.error(msg)
        raise RuntimeError(msg)
Example #21
0
    def _filter_types_by_policy(self, context, types):
        def _allowed(_type):
            return policy.plugin_allowed(
                self.policy_enforcer, context, self.plugins[_type].obj)

        allowed_types = list(filter(_allowed, types))
        if not allowed_types:
            disallowed_str = ", ".join(sorted(types))
            msg = _("There are no resource types accessible to you to serve "
                    "your request. You do not have access to the "
                    "following resource types: %s") % disallowed_str
            raise webob.exc.HTTPForbidden(explanation=msg)
        return allowed_types
Example #22
0
def parse_valid_host_port(host_port):
    """
    Given a "host:port" string, attempts to parse it as intelligently as
    possible to determine if it is valid. This includes IPv6 [host]:port form,
    IPv4 ip:port form, and hostname:port or fqdn:port form.

    Invalid inputs will raise a ValueError, while valid inputs will return
    a (host, port) tuple where the port will always be of type int.
    """

    try:
        try:
            host, port = netutils.parse_host_port(host_port)
        except Exception:
            raise ValueError(_('Host and port "%s" is not valid.') % host_port)

        if not netutils.is_valid_port(port):
            raise ValueError(_('Port "%s" is not valid.') % port)

        # First check for valid IPv6 and IPv4 addresses, then a generic
        # hostname. Failing those, if the host includes a period, then this
        # should pass a very generic FQDN check. The FQDN check for letters at
        # the tail end will weed out any hilariously absurd IPv4 addresses.

        if not (netutils.is_valid_ipv6(host) or netutils.is_valid_ipv4(host) or
                is_valid_hostname(host) or is_valid_fqdn(host)):
            raise ValueError(_('Host "%s" is not valid.') % host)

    except Exception as ex:
        raise ValueError(_('%s '
                           'Please specify a host:port pair, where host is an '
                           'IPv4 address, IPv6 address, hostname, or FQDN. If '
                           'using an IPv6 address, enclose it in brackets '
                           'separately from the port (i.e., '
                           '"[fe80::a:b:c]:9876").') % ex)

    return (host, int(port))
Example #23
0
 def anticipate_respawn(children):
     while children:
         pid, status = os.wait()
         if pid in children:
             (pid_file, server, args) = children.pop(pid)
             running = os.path.exists(pid_file)
             one_second_ago = time.time() - 1
             bouncing = (running and
                         os.path.getmtime(pid_file) >= one_second_ago)
             if running and not bouncing:
                 args = (pid_file, server, args)
                 new_pid = do_start('Respawn', *args)
                 children[new_pid] = args
             else:
                 rsn = 'bouncing' if bouncing else 'deliberately stopped'
                 print(_('Suppressed respawn as %(serv)s was %(rsn)s.')
                       % {'serv': server, 'rsn': rsn})
Example #24
0
    def _get_es_query(self, context, query, resource_types,
                      all_projects=False):
        is_admin = context.is_admin
        ignore_rbac = is_admin and all_projects

        type_and_rbac_filters = []
        for resource_type in resource_types:
            plugin = self.plugins[resource_type].obj

            try:
                plugin_filter = plugin.get_query_filters(
                    context, ignore_rbac=ignore_rbac)

                type_and_rbac_filters.append(plugin_filter)
            except Exception as e:
                msg = _("Error processing %s RBAC filter") % resource_type
                LOG.error("Failed to retrieve RBAC filters "
                          "from search plugin "
                          "%(ext)s: %(e)s" %
                          {'ext': plugin.name, 'e': e})
                raise webob.exc.HTTPInternalServerError(explanation=msg)

        role_filter = {'term': {searchlight.elasticsearch.ROLE_USER_FIELD:
                                context.user_role_filter}}

        # Create a filter query for the role filter; RBAC filters are added
        # in the next step
        es_query = {
            'bool': {
                'filter': {
                    'bool': {
                        'must': role_filter
                    }
                },
                'must': query
            }
        }

        if type_and_rbac_filters:
            # minimum_should_match: 1 is assumed in filter context,
            # but I'm including it explicitly so nobody spends an
            # hour scouring the documentation to check
            es_query['bool']['filter']['bool'].update(
                {'should': type_and_rbac_filters,
                 'minimum_should_match': 1})
        return {'query': es_query}
Example #25
0
    def get_full_mapping(self):
        """Gets the full mapping doc for this type, including children. This
        returns a child-first (depth-first) generator.
        """
        # Assemble the children!
        for child_plugin in self.child_plugins:
            for plugin_type, mapping in child_plugin.get_full_mapping():
                yield plugin_type, mapping

        type_mapping = self.get_mapping()

        # Add common mapping fields
        # ROLE_USER_FIELD is required for RBAC by all plugins
        type_mapping['properties'][ROLE_USER_FIELD] = {
            'type': 'string',
            'index': 'not_analyzed',
            'include_in_all': False
        }

        # Add region name
        if self.include_region_name:
            type_mapping['properties']['region_name'] = {
                'type': 'string',
                'index': 'not_analyzed',
            }

        if 'updated_at' not in type_mapping['properties'].keys():
            type_mapping['properties']['updated_at'] = {'type': 'date'}

        if self.mapping_use_doc_values:
            helper.IndexingHelper.apply_doc_values(type_mapping)

        expected_parent_type = self.parent_plugin_type()
        mapping_parent = type_mapping.get('_parent', None)
        if mapping_parent:
            if mapping_parent['type'] != expected_parent_type:
                raise exception.IndexingException(
                    _("Mapping for '%(doc_type)s' contains a _parent "
                      "'%(actual)s' that doesn't match '%(expected)s'") %
                    {"doc_type": self.document_type,
                     "actual": mapping_parent['type'],
                     "expected": expected_parent_type})
        elif expected_parent_type:
            type_mapping['_parent'] = {'type': expected_parent_type}

        yield self.document_type, type_mapping
Example #26
0
    def search(self, req, query, index=None, doc_type=None,
               from_=0, size=None, **kwargs):
        """Supported kwargs:
        :param _source:
        :param _source_include:
        :param _source_exclude:
        :return:
        """
        if size is None:
            size = CONF.limit_param_default

        try:
            search_repo = self.gateway.get_catalog_search_repo(req.context)
            result = search_repo.search(index,
                                        doc_type,
                                        query,
                                        from_,
                                        size,
                                        ignore_unavailable=True,
                                        **kwargs)

            hits = result.get('hits', {}).get('hits', [])
            try:
                # Note that there's an assumption that the plugin resource
                # type is always the same as the document type. If this is not
                # the case in future, a reverse lookup's needed
                for hit in hits:
                    plugin = self.plugins[hit['_type']].obj
                    plugin.filter_result(hit, req.context)
            except KeyError as e:
                raise Exception("No registered plugin for type %s" % e.message)
            return result
        except exception.Forbidden as e:
            raise webob.exc.HTTPForbidden(explanation=e.msg)
        except exception.NotFound as e:
            raise webob.exc.HTTPNotFound(explanation=e.msg)
        except exception.Duplicate as e:
            raise webob.exc.HTTPConflict(explanation=e.msg)
        except es_exc.RequestError:
            msg = _("Query malformed or search parse failure, please check "
                    "the syntax")
            raise webob.exc.HTTPBadRequest(explanation=msg)
        except Exception as e:
            LOG.error(encodeutils.exception_to_unicode(e))
            raise webob.exc.HTTPInternalServerError()
Example #27
0
def fix_greendns_ipv6():
    if dnspython_installed:
        # All of this is because if dnspython is present in your environment
        # then eventlet monkeypatches socket.getaddrinfo() with an
        # implementation which doesn't work for IPv6. What we're checking here
        # is that the magic environment variable was set when the import
        # happened.
        nogreendns = 'EVENTLET_NO_GREENDNS'
        flag = os.environ.get(nogreendns, '')
        if 'eventlet' in sys.modules and not strutils.bool_from_string(flag):
            msg = _("It appears that the eventlet module has been "
                    "imported prior to setting %s='yes'. It is currently "
                    "necessary to disable eventlet.greendns "
                    "if using ipv6 since eventlet.greendns currently "
                    "breaks with ipv6 addresses. Please ensure that "
                    "eventlet is not imported prior to this being set.")
            raise ImportError(msg % (nogreendns))

        os.environ[nogreendns] = 'yes'
Example #28
0
    def _validate_aggregations(self, context, aggregations):
        if aggregations:
            # Check aggregations against policy
            try:
                self.policy_enforcer.enforce(context,
                                             'search:query:aggregations',
                                             context.policy_target)
            except exception.Forbidden as e:
                raise webob.exc.HTTPForbidden(explanation=e.msg)

            # Reject any requests including the 'global' aggregation type
            # because it bypasses RBAC. 'global' aggregations can only occur
            # at the top level, so we only need to check that level
            for agg_name, agg_definition in aggregations.items():
                if 'global' in agg_definition:
                    msg = _(
                        "Aggregation '%s' contains the 'global' aggregation "
                        "which is not allowed") % agg_name
                    LOG.error(msg)
                    raise webob.exc.HTTPForbidden(explanation=msg)
        return aggregations
Example #29
0
    def topics_and_exchanges(self):
        topics_exchanges = set()
        for plugin_type, plugin in self.plugins.items():
            try:
                handler = plugin.obj.get_notification_handler()
                if handler:
                    topic_exchanges = (
                        handler.get_notification_topics_exchanges())
                    for plugin_topic in topic_exchanges:
                        if isinstance(plugin_topic, six.string_types):
                            raise Exception(
                                _("Plugin %s should return a list of topic"
                                  "exchange pairs") %
                                plugin.__class__.__name__)
                        topics_exchanges.add(plugin_topic)
            except Exception as e:
                LOG.error("Failed to retrieve notification topic(s)"
                          " and exchanges from search plugin "
                          "%(ext)s: %(e)s" %
                          {'ext': plugin.name, 'e': e})

        return topics_exchanges
    def process_request(self, req):
        """Try to find a version first in the accept header, then the URL"""
        msg = _("Determining version of request: %(method)s %(path)s"
                " Accept: %(accept)s")
        args = {'method': req.method, 'path': req.path, 'accept': req.accept}
        LOG.debug(msg % args)

        LOG.debug("Using url versioning")
        # Remove version in url so it doesn't conflict later
        req_version = self._pop_path_info(req)

        try:
            version = self._match_version_string(req_version)
        except ValueError:
            LOG.warning(_LW("Unknown version. Returning version choices."))
            return self.versions_app

        req.environ['api.version'] = version
        req.path_info = ''.join(('/v', str(version), req.path_info))
        LOG.debug("Matched version: v%d", version)
        LOG.debug('new path %s', req.path_info)
        return None
Example #31
0
class VersionedNotificationMismatch(SearchlightException):
    message = _("Provided notification version "
                "%(provided_maj)s.%(provided_min)s did not match expected "
                "%(expected_maj)s.%(expected_min)s for %(type)s")
Example #32
0
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
import webob.exc

from searchlight.api import policy
from searchlight.common import wsgi
import searchlight.context
from searchlight.i18n import _, _LW

context_opts = [
    cfg.BoolOpt('owner_is_tenant',
                default=True,
                help=_('When true, this option sets the owner of an image '
                       'to be the tenant. Otherwise, the owner of the '
                       ' image will be the authenticated user issuing the '
                       'request.')),
    cfg.StrOpt('admin_role',
               default='admin',
               help=_('Role used to identify an authenticated user as '
                      'administrator.')),
    cfg.BoolOpt('allow_anonymous_access',
                default=False,
                help=_('Allow unauthenticated users to access the API with '
                       'read-only privileges. This only applies when using '
                       'ContextMiddleware.')),
]

CONF = cfg.CONF
CONF.register_opts(context_opts)
Example #33
0
import tempfile

from oslo_concurrency import lockutils
from oslo_config import cfg
from oslo_middleware import cors
from oslo_policy import policy
from paste import deploy

from searchlight.i18n import _
from searchlight.version import version_info as version

paste_deploy_opts = [
    cfg.StrOpt('flavor',
               help=_('Partial name of a pipeline in your paste configuration '
                      'file with the service name removed. For example, if '
                      'your paste section name is '
                      '[pipeline:searchlight-api-keystone] use the value '
                      '"keystone"')),
    cfg.StrOpt('api_paste_config',
               default="api-paste.ini",
               help=_('The API paste config file to use.')),
]

common_opts = [
    cfg.IntOpt('limit_param_default',
               default=25,
               help=_('Default value for the number of items returned by a '
                      'request if not specified explicitly in the request')),
    cfg.IntOpt('api_limit_max',
               default=1000,
               help=_('Maximum permissible number of items that could be '
Example #34
0
from searchlight.common import exception
from searchlight.common import loggers
from searchlight.common import utils
from searchlight import i18n
from searchlight.i18n import _

try:
    from webob.acceptparse import AcceptLanguageValidHeader  # noqa
    USING_WEBOB_1_8 = True
except ImportError:
    USING_WEBOB_1_8 = False

bind_opts = [
    cfg.HostAddressOpt('bind_host',
                       default='0.0.0.0',
                       help=_('Address to bind the server.  Useful when '
                              'selecting a particular network interface.')),
    cfg.IntOpt('bind_port',
               help=_('The port on which the server will listen.')),
]

socket_opts = [
    cfg.IntOpt('backlog',
               default=4096,
               help=_('The backlog value that will be used when creating the '
                      'TCP listener socket.')),
    cfg.IntOpt('tcp_keepidle',
               default=600,
               help=_('The value for the socket option TCP_KEEPIDLE.  This is '
                      'the time in seconds that the connection must be idle '
                      'before TCP starts sending keepalive probes.')),
    cfg.StrOpt('ca_file',
Example #35
0
class InvalidJsonPatchBody(JsonPatchException):
    message = _("The provided body %(body)s is invalid "
                "under given schema: %(schema)s")
Example #36
0
 def _get_request_body(self, request):
     output = super(RequestDeserializer, self).default(request)
     if 'body' not in output:
         msg = _('Body expected in request.')
         raise webob.exc.HTTPBadRequest(explanation=msg)
     return output['body']
Example #37
0
 def _validate_resource_type(self, resource_type):
     if resource_type not in self._get_available_resource_types():
         msg = _("Resource type '%s' is not supported.") % resource_type
         raise webob.exc.HTTPBadRequest(explanation=msg)
     return resource_type
Example #38
0
class WorkerCreationFailure(SearchlightException):
    message = _("Server worker creation failed: %(reason)s.")
Example #39
0
class InvalidContentType(SearchlightException):
    message = _("Invalid content type %(content_type)s")
Example #40
0
class SchemaLoadError(SearchlightException):
    message = _("Unable to load schema: %(reason)s")
Example #41
0
 def from_json(self, datastring):
     try:
         return jsonutils.loads(datastring, object_hook=self._sanitizer)
     except ValueError:
         msg = _('Malformed JSON in request body.')
         raise webob.exc.HTTPBadRequest(explanation=msg)
Example #42
0
class InvalidObject(SearchlightException):
    message = _("Provided object does not match schema "
                "'%(schema)s': %(reason)s")
Example #43
0
class SIGHUPInterrupt(SearchlightException):
    message = _("System SIGHUP signal received.")
Example #44
0
class JsonPatchException(SearchlightException):
    message = _("Invalid jsonpatch request")
Example #45
0
class InvalidAPIVersionProvided(SearchlightException):
    message = _("The provided API version is not supported, "
                "the current available version range for %(service)s "
                "is: from %(min_version)s to %(max_version)s.")
Example #46
0
class ReservedProperty(Forbidden):
    message = _("Attribute '%(property)s' is reserved.")
Example #47
0
class IndexingException(SearchlightException):
    message = _("An error occurred during index creation or initial loading")
Example #48
0
class InvalidPropertyProtectionConfiguration(Invalid):
    message = _("Invalid configuration in property protection file.")
Example #49
0
 def _check_allowed(cls, query):
     for key in cls._disallowed_properties:
         if key in query:
             msg = _("Attribute '%s' is read-only.") % key
             raise webob.exc.HTTPForbidden(explanation=msg)
Example #50
0
class Invalid(SearchlightException):
    message = _("Data supplied was not valid.")
Example #51
0
    def search(self, request):
        body = self._get_request_body(request)
        self._check_allowed(body)
        query = body.pop('query', {"match_all": {}})
        indices = body.pop('index', None)
        types = body.pop('type', None)
        _source = body.pop('_source', None)
        offset = body.pop('offset', None)
        from_ = body.pop('from', None)
        limit = body.pop('limit', None)
        size = body.pop('size', None)
        highlight = body.pop('highlight', None)
        aggregations = body.pop('aggregations', None)
        if not aggregations:
            aggregations = body.pop('aggs', None)
        elif 'aggs' in body:
            raise webob.exc.HTTPBadRequest("A request cannot include both "
                                           "'aggs' and 'aggregations'")
        sort_order = body.pop('sort', None)
        # all_projects will determine whether an admin sees
        # filtered results or not
        all_projects = body.pop('all_projects', False)
        # Return _version with results?
        version = body.pop('version', None)

        aggregations = self._validate_aggregations(request.context,
                                                   aggregations)

        available_types = self._get_available_types()
        if not types:
            types = available_types
        else:
            if not isinstance(types, (list, tuple)):
                types = [types]

            for requested_type in types:
                if requested_type not in available_types:
                    msg = _("Resource type '%s' is not in the list of enabled "
                            "plugins") % requested_type
                    raise webob.exc.HTTPBadRequest(explanation=msg)

        # Filter the list by policy before determining which indices to use
        types = self._filter_types_by_policy(request.context, types)

        available_indices = self._get_available_indices(types)
        if not indices:
            indices = available_indices
        else:
            if not isinstance(indices, (list, tuple)):
                indices = [indices]

            for requested_index in indices:
                if requested_index not in available_indices:
                    msg = _("Index '%s' is not in the list of enabled "
                            "plugins") % requested_index
                    raise webob.exc.HTTPBadRequest(explanation=msg)

        if not isinstance(indices, (list, tuple)):
            indices = [indices]

        query_params = {
            'query':
            self._get_es_query(request.context,
                               query,
                               types,
                               all_projects=all_projects)
        }

        # Apply an additional restriction to elasticsearch to speed things up
        # in addition to the RBAC filters
        query_params['index'] = indices
        query_params['doc_type'] = types

        # Don't set query_params['_source'] any more; we ALWAYS want to
        # exclude the role user field, so if the query specifies just
        # _source=<string>, put that in _source_include
        source_exclude = [searchlight.elasticsearch.ROLE_USER_FIELD]

        if _source is not None:
            if isinstance(_source, dict):
                if 'include' in _source:
                    query_params['_source_include'] = _source['include']
                if 'exclude' in _source:
                    if isinstance(_source['exclude'], six.text_type):
                        source_exclude.append(_source['exclude'])
                    else:
                        source_exclude.extend(_source['exclude'])
            elif isinstance(_source, (list, six.text_type)):
                query_params['_source_include'] = _source
            else:
                msg = _("'_source' must be a string, dict or list")
                raise webob.exc.HTTPBadRequest(explanation=msg)

        query_params['_source_exclude'] = source_exclude

        from_ = self._validate_offset(offset, from_)
        if from_ is not None:
            query_params['from_'] = from_

        size = self._validate_limit(limit, size)
        if size is not None:
            query_params['size'] = size

        if highlight is not None:
            self._set_highlight_queries(highlight, query)
            query_params['query']['highlight'] = highlight

        if aggregations is not None:
            query_params['query']['aggregations'] = aggregations

        if sort_order is not None:
            query_params['query']['sort'] = self._get_sort_order(sort_order)

        if version is not None:
            query_params['version'] = version

        return query_params
Example #52
0
class Forbidden(SearchlightException):
    message = _("You are not authorized to complete this action.")
Example #53
0
class InvalidJsonPatchPath(JsonPatchException):
    message = _("The provided path '%(path)s' is invalid: %(explanation)s")

    def __init__(self, message=None, *args, **kwargs):
        self.explanation = kwargs.get("explanation")
        super(InvalidJsonPatchPath, self).__init__(message, *args, **kwargs)
Example #54
0
class Duplicate(SearchlightException):
    message = _("An object with the same identifier already exists.")
Example #55
0
# NOTE(bourke): The default dict_type is collections.OrderedDict in py27, but
# we must set manually for compatibility with py26
# SafeConfigParser was deprecated in Python 3.2
if six.PY3:
    CONFIG = configparser.ConfigParser(dict_type=OrderedDict)
else:
    CONFIG = configparser.SafeConfigParser(dict_type=OrderedDict)
LOG = logging.getLogger(__name__)

property_opts = [
    cfg.StrOpt('property_protection_file',
               help=_('The location of the property protection file.'
                      'This file contains the rules for property protections '
                      'and the roles/policies associated with it. If this '
                      'config value is not specified, by default, property '
                      'protections won\'t be enforced. If a value is '
                      'specified and the file is not found, then the '
                      'searchlight-api service will not start.')),
    cfg.StrOpt('property_protection_rule_format',
               default='roles',
               choices=('roles', 'policies'),
               help=_('This config value indicates whether "roles" or '
                      '"policies" are used in the property protection file.')),
]

CONF = cfg.CONF
CONF.register_opts(property_opts)

# NOTE (spredzy): Due to the particularly lengthy name of the exception
# and the number of occurrence it is raise in this file, a variable is
Example #56
0
class NotFound(SearchlightException):
    message = _("An object with the specified identifier was not found.")
Example #57
0
#    under the License.

from http import client as http_client

from oslo_config import cfg
from oslo_serialization import jsonutils
import webob.dec

from searchlight.common import wsgi
from searchlight.i18n import _

versions_opts = [
    cfg.StrOpt('public_endpoint',
               help=_('Public url to use for versions endpoint. The default '
                      'is None, which will use the request\'s host_url '
                      'attribute to populate the URL base. If Searchlight is '
                      'operating behind a proxy, you will want to change '
                      'this to represent the proxy\'s URL.')),
]

CONF = cfg.CONF
CONF.register_opts(versions_opts, group='api')


class Controller(object):
    """A wsgi controller that reports which API versions are supported."""
    def index(self, req):
        """Respond to a request for all OpenStack API versions."""
        def build_version_object(version, path, status):
            url = CONF.api.public_endpoint or req.host_url
            url = url.rstrip("/")
Example #58
0
def do_start(verb, pid_file, server, args):
    if verb != 'Respawn' and pid_file == CONF.pid_file:
        for pid_file, pid in pid_files(server, pid_file):
            if os.path.exists('/proc/%s' % pid):
                print(
                    _("%(serv)s appears to already be running: %(pid)s") % {
                        'serv': server,
                        'pid': pid_file
                    })
                return
            else:
                print(_("Removing stale pid file %s") % pid_file)
                os.unlink(pid_file)

        try:
            resource.setrlimit(resource.RLIMIT_NOFILE,
                               (MAX_DESCRIPTORS, MAX_DESCRIPTORS))
            resource.setrlimit(resource.RLIMIT_DATA, (MAX_MEMORY, MAX_MEMORY))
        except ValueError:
            print(
                _('Unable to increase file descriptor limit.  '
                  'Running as non-root?'))
        os.environ['PYTHON_EGG_CACHE'] = '/tmp'

    def write_pid_file(pid_file, pid):
        with open(pid_file, 'w') as fp:
            fp.write('%d\n' % pid)

    def redirect_to_null(fds):
        with open(os.devnull, 'r+b') as nullfile:
            for desc in fds:  # close fds
                try:
                    os.dup2(nullfile.fileno(), desc)
                except OSError:
                    pass

    def redirect_to_syslog(fds, server):
        log_cmd = 'logger'
        log_cmd_params = '-t "%s[%d]"' % (server, os.getpid())
        process = subprocess.Popen([log_cmd, log_cmd_params],
                                   stdin=subprocess.PIPE)
        for desc in fds:  # pipe to logger command
            try:
                os.dup2(process.stdin.fileno(), desc)
            except OSError:
                pass

    def redirect_stdio(server, capture_output):
        input = [sys.stdin.fileno()]
        output = [sys.stdout.fileno(), sys.stderr.fileno()]

        redirect_to_null(input)
        if capture_output:
            redirect_to_syslog(output, server)
        else:
            redirect_to_null(output)

    @gated_by(CONF.capture_output)
    def close_stdio_on_exec():
        fds = [sys.stdin.fileno(), sys.stdout.fileno(), sys.stderr.fileno()]
        for desc in fds:  # set close on exec flag
            fcntl.fcntl(desc, fcntl.F_SETFD, fcntl.FD_CLOEXEC)

    def launch(pid_file, conf_file=None, capture_output=False, await_time=0):
        args = [server]
        if conf_file:
            args += ['--config-file', conf_file]
            msg = (_('%(verb)sing %(serv)s with %(conf)s') % {
                'verb': verb,
                'serv': server,
                'conf': conf_file
            })
        else:
            msg = (_('%(verb)sing %(serv)s') % {'verb': verb, 'serv': server})
        print(msg)

        close_stdio_on_exec()

        pid = os.fork()
        if pid == 0:
            os.setsid()
            redirect_stdio(server, capture_output)
            try:
                os.execlp('%s' % server, *args)
            except OSError as e:
                msg = (_('unable to launch %(serv)s. Got error: %(e)s') % {
                    'serv': server,
                    'e': e
                })
                sys.exit(msg)
            sys.exit(0)
        else:
            write_pid_file(pid_file, pid)
            await_child(pid, await_time)
            return pid

    @gated_by(CONF.await_child)
    def await_child(pid, await_time):
        bail_time = time.time() + await_time
        while time.time() < bail_time:
            reported_pid, status = os.waitpid(pid, os.WNOHANG)
            if reported_pid == pid:
                global exitcode
                exitcode = os.WEXITSTATUS(status)
                break
            time.sleep(0.05)

    conf_file = None
    if args and os.path.exists(args[0]):
        conf_file = os.path.abspath(os.path.expanduser(args[0]))

    return launch(pid_file, conf_file, CONF.capture_output, CONF.await_child)