コード例 #1
0
    def lookup(self, key):
        if key == 'entities' or key is None:
            return self._entities()

        bkey = six.b(key)
        if bkey in self.objects:
            return [self.objects.get(bkey)]

        if bkey in self.parts:
            res = []
            part = self.parts.get(bkey)
            for item in part['items']:
                res.extend(self.lookup(item))
            return res

        key = self._prep_key(key)
        qp = QueryParser("object_id", schema=self.schema)
        q = qp.parse(key)
        lst = set()
        with self.index.searcher() as searcher:
            results = searcher.search(q, limit=None)
            for result in results:
                e = self.objects.get(result['object_id'], None)
                if e is not None:
                    lst.add(e)

        return b2u(list(lst))
コード例 #2
0
 def attribute(self, a):
     if a in ATTRS_INV:
         n = ATTRS_INV[a]
         ix = self.storage.open_index()
         with ix.searcher() as searcher:
             return b2u(list(searcher.lexicon(n)))
     else:
         return []
コード例 #3
0
    def _entities(self):
        lst = set()
        for ref_data in self.parts.values():
            for ref in ref_data['items']:
                e = self.objects.get(ref, None)
                if e is not None:
                    lst.add(e)

        return b2u(list(lst))
コード例 #4
0
    def _reindex(self):
        log.debug("indexing the store...")
        self._last_index_time = datetime.now()
        seen = set()
        refs = set([b2u(s) for s in self.objects.keys()])
        parts = self.parts.values()
        for ref in refs:
            for part in parts:
                if ref in part['items']:
                    seen.add(ref)

        ix = self.storage.open_index()
        lock = ix.lock("reindex")
        try:
            log.debug("waiting for index lock")
            lock.acquire(True)
            log.debug("got index lock")
            with ix.writer() as writer:
                for ref in refs:
                    if ref not in seen:
                        log.debug("removing unseen ref {}".format(ref))
                        del self.objects[ref]
                        del self.parts[ref]

                log.debug("updating index")
                for e in self.objects.values():
                    info = self._index_prep(entity_simple_info(e))
                    ref = object_id(e)
                    writer.add_document(object_id=ref, **info)

                writer.mergetype = CLEAR
        finally:
            try:
                log.debug("releasing index lock")
                lock.release()
            except ThreadError as ex:
                pass
コード例 #5
0
 def test_b2u(self):
     assert(int(b2u(b'1')) == 1)
     assert(b2u('kaka') == 'kaka')
コード例 #6
0
ファイル: api.py プロジェクト: clarin-eric/pyFF
def process_handler(request: Request) -> Response:
    """
    The main request handler for pyFF. Implements API call hooks and content negotiation.

    :param request: the HTTP request object
    :return: the data to send to the client
    """
    _ctypes = {
        'xml': 'application/samlmetadata+xml;application/xml;text/xml',
        'json': 'application/json'
    }

    def _d(x: Optional[str],
           do_split: bool = True) -> Tuple[Optional[str], Optional[str]]:
        """ Split a path into a base component and an extension. """
        if x is not None:
            x = x.strip()

        if x is None or len(x) == 0:
            return None, None

        if '.' in x:
            (pth, dot, extn) = x.rpartition('.')
            assert dot == '.'
            if extn in _ctypes:
                return pth, extn

        return x, None

    log.debug(f'Processing request: {request}')

    if request.matchdict is None:
        raise exc.exception_response(400)

    if request.body:
        try:
            request.matchdict.update(request.json_body)
        except ValueError as ex:
            pass

    entry = request.matchdict.get('entry', 'request')
    path_elem = list(request.matchdict.get('path', []))
    match = request.params.get('q', request.params.get('query', None))

    # Enable matching on scope.
    match = match.split(
        '@').pop() if match and not match.endswith('@') else match
    log.debug("match={}".format(match))

    if not path_elem:
        path_elem = ['entities']

    alias = path_elem.pop(0)
    path = '/'.join(path_elem)

    # Ugly workaround bc WSGI drops double-slashes.
    path = path.replace(':/', '://')

    msg = "handling entry={}, alias={}, path={}"
    log.debug(msg.format(entry, alias, path))

    pfx = None
    if 'entities' not in alias:
        pfx = request.registry.aliases.get(alias, None)
        if pfx is None:
            raise exc.exception_response(404)

    # content_negotiation_policy is one of three values:
    # 1. extension - current default, inspect the path and if it ends in
    #    an extension, e.g. .xml or .json, always strip off the extension to
    #    get the entityID and if no accept header or a wildcard header, then
    #    use the extension to determine the return Content-Type.
    #
    # 2. adaptive - only if no accept header or if a wildcard, then inspect
    #    the path and if it ends in an extension strip off the extension to
    #    get the entityID and use the extension to determine the return
    #    Content-Type.
    #
    # 3. header - future default, do not inspect the path for an extension and
    #    use only the Accept header to determine the return Content-Type.
    policy = config.content_negotiation_policy

    # TODO - sometimes the client sends > 1 accept header value with ','.
    accept = str(request.accept).split(',')[0]
    valid_accept = accept and not ('application/*' in accept
                                   or 'text/*' in accept or '*/*' in accept)

    new_path: Optional[str] = path
    path_no_extension, extension = _d(new_path, True)
    accept_from_extension = accept
    if extension:
        accept_from_extension = _ctypes.get(extension, accept)

    if policy == 'extension':
        new_path = path_no_extension
        if not valid_accept:
            accept = accept_from_extension
    elif policy == 'adaptive':
        if not valid_accept:
            new_path = path_no_extension
            accept = accept_from_extension

    if not accept:
        log.warning('Could not determine accepted response type')
        raise exc.exception_response(400)

    q: Optional[str]
    if pfx and new_path:
        q = f'{{{pfx}}}{new_path}'
        new_path = f'/{alias}/{new_path}'
    else:
        q = new_path

    try:
        accepter = MediaAccept(accept)
        for p in request.registry.plumbings:
            state = {
                entry: True,
                'headers': {
                    'Content-Type': None
                },
                'accept': accepter,
                'url': request.current_route_url(),
                'select': q,
                'match': match.lower() if match else match,
                'path': new_path,
                'stats': {},
            }

            r = p.process(request.registry.md,
                          state=state,
                          raise_exceptions=True,
                          scheduler=request.registry.scheduler)
            log.debug(f'Plumbing process result: {r}')
            if r is None:
                r = []

            response = Response()
            _headers = state.get('headers', {})
            response.headers.update(_headers)
            ctype = _headers.get('Content-Type', None)
            if not ctype:
                r, t = _fmt(r, accepter)
                ctype = t

            response.text = b2u(r)
            response.size = len(r)
            response.content_type = ctype
            cache_ttl = int(state.get('cache', 0))
            response.expires = datetime.now() + timedelta(seconds=cache_ttl)
            return response
    except ResourceException as ex:
        import traceback

        log.debug(traceback.format_exc())
        log.warning(f'Exception from processing pipeline: {ex}')
        raise exc.exception_response(409)
    except BaseException as ex:
        import traceback

        log.debug(traceback.format_exc())
        log.error(f'Exception from processing pipeline: {ex}')
        raise exc.exception_response(500)

    if request.method == 'GET':
        raise exc.exception_response(404)
コード例 #7
0
 def attributes(self):
     return b2u(list(self._attributes()))
コード例 #8
0
 def _attributes(self):
     ix = self.storage.open_index()
     with ix.reader() as reader:
         for n in reader.indexed_field_names():
             if n in ATTRS:
                 yield b2u(ATTRS[n])
コード例 #9
0
 def collections(self):
     return [b2u(ref) for ref in self.parts.keys()]
コード例 #10
0
 def _unpickle(self, x):
     return json.loads(b2u(x))
コード例 #11
0
 def test_b2u(self):
     assert int(b2u(b'1')) == 1
     assert b2u('kaka') == 'kaka'
コード例 #12
0
ファイル: samlmd.py プロジェクト: clarin-eric/pyFF
def entitiesdescriptor(
    entities,
    name,
    lookup_fn=None,
    cache_duration=None,
    valid_until=None,
    validate=True,
    filter_invalid=True,
    copy=True,
    nsmap=None,
):
    """
    :param lookup_fn: a function used to lookup entities by name - set to None to skip resolving
    :param entities: a set of entities specifiers (lookup is used to find entities from this set)
    :param name: the @Name attribute
    :param cache_duration: an XML timedelta expression, eg PT1H for 1hr
    :param valid_until: a relative time eg 2w 4d 1h for 2 weeks, 4 days and 1hour from now.
    :param copy: set to False to avoid making a copy of all the entities in list. This may be dangerous.
    :param validate: set to False to skip schema validation of the resulting EntitiesDescriptor element. This is dangerous!
    :param filter_invalid: remove invalid EntitiesDescriptor elements from aggregate
    :param nsmap: additional namespace definitions to include in top level EntitiesDescriptor element

    Produce an EntityDescriptors set from a list of entities. Optional Name, cacheDuration and validUntil are affixed.
    """

    if nsmap is None:
        nsmap = dict()

    nsmap.update(NS)

    if lookup_fn is not None:
        entities = resolve_entities(entities, lookup_fn=lookup_fn)

    for entity in entities:
        nsmap.update(entity.nsmap)

    log.debug("selecting %d entities before validation" % len(entities))

    attrs = dict(Name=name, nsmap=nsmap)
    if cache_duration is not None:
        attrs['cacheDuration'] = cache_duration
    if valid_until is not None:
        attrs['validUntil'] = valid_until
    t = etree.Element("{%s}EntitiesDescriptor" % NS['md'], **attrs)
    for entity in entities:
        ent_insert = entity
        if copy:
            ent_insert = deepcopy(ent_insert)
        t.append(ent_insert)

    if config.devel_write_xml_to_file:
        import os

        with open("/tmp/pyff_entities_out-{}.xml".format(os.getpid()),
                  "w") as fd:
            fd.write(b2u(dumptree(t)))

    if validate:
        validation_errors = dict()
        t = filter_or_validate(t,
                               filter_invalid=filter_invalid,
                               base_url=name,
                               source="request",
                               validation_errors=validation_errors)

        for base_url, err in validation_errors.items():
            log.error("Validation error: @ {}: {}".format(base_url, err))

    return t