def lookup(self, key): if key == 'entities' or key is None: return self._entities() bkey = six.b(key) if bkey in self.objects: return [self.objects.get(bkey)] if bkey in self.parts: res = [] part = self.parts.get(bkey) for item in part['items']: res.extend(self.lookup(item)) return res key = self._prep_key(key) qp = QueryParser("object_id", schema=self.schema) q = qp.parse(key) lst = set() with self.index.searcher() as searcher: results = searcher.search(q, limit=None) for result in results: e = self.objects.get(result['object_id'], None) if e is not None: lst.add(e) return b2u(list(lst))
def attribute(self, a): if a in ATTRS_INV: n = ATTRS_INV[a] ix = self.storage.open_index() with ix.searcher() as searcher: return b2u(list(searcher.lexicon(n))) else: return []
def _entities(self): lst = set() for ref_data in self.parts.values(): for ref in ref_data['items']: e = self.objects.get(ref, None) if e is not None: lst.add(e) return b2u(list(lst))
def _reindex(self): log.debug("indexing the store...") self._last_index_time = datetime.now() seen = set() refs = set([b2u(s) for s in self.objects.keys()]) parts = self.parts.values() for ref in refs: for part in parts: if ref in part['items']: seen.add(ref) ix = self.storage.open_index() lock = ix.lock("reindex") try: log.debug("waiting for index lock") lock.acquire(True) log.debug("got index lock") with ix.writer() as writer: for ref in refs: if ref not in seen: log.debug("removing unseen ref {}".format(ref)) del self.objects[ref] del self.parts[ref] log.debug("updating index") for e in self.objects.values(): info = self._index_prep(entity_simple_info(e)) ref = object_id(e) writer.add_document(object_id=ref, **info) writer.mergetype = CLEAR finally: try: log.debug("releasing index lock") lock.release() except ThreadError as ex: pass
def test_b2u(self): assert(int(b2u(b'1')) == 1) assert(b2u('kaka') == 'kaka')
def process_handler(request: Request) -> Response: """ The main request handler for pyFF. Implements API call hooks and content negotiation. :param request: the HTTP request object :return: the data to send to the client """ _ctypes = { 'xml': 'application/samlmetadata+xml;application/xml;text/xml', 'json': 'application/json' } def _d(x: Optional[str], do_split: bool = True) -> Tuple[Optional[str], Optional[str]]: """ Split a path into a base component and an extension. """ if x is not None: x = x.strip() if x is None or len(x) == 0: return None, None if '.' in x: (pth, dot, extn) = x.rpartition('.') assert dot == '.' if extn in _ctypes: return pth, extn return x, None log.debug(f'Processing request: {request}') if request.matchdict is None: raise exc.exception_response(400) if request.body: try: request.matchdict.update(request.json_body) except ValueError as ex: pass entry = request.matchdict.get('entry', 'request') path_elem = list(request.matchdict.get('path', [])) match = request.params.get('q', request.params.get('query', None)) # Enable matching on scope. match = match.split( '@').pop() if match and not match.endswith('@') else match log.debug("match={}".format(match)) if not path_elem: path_elem = ['entities'] alias = path_elem.pop(0) path = '/'.join(path_elem) # Ugly workaround bc WSGI drops double-slashes. path = path.replace(':/', '://') msg = "handling entry={}, alias={}, path={}" log.debug(msg.format(entry, alias, path)) pfx = None if 'entities' not in alias: pfx = request.registry.aliases.get(alias, None) if pfx is None: raise exc.exception_response(404) # content_negotiation_policy is one of three values: # 1. extension - current default, inspect the path and if it ends in # an extension, e.g. .xml or .json, always strip off the extension to # get the entityID and if no accept header or a wildcard header, then # use the extension to determine the return Content-Type. # # 2. adaptive - only if no accept header or if a wildcard, then inspect # the path and if it ends in an extension strip off the extension to # get the entityID and use the extension to determine the return # Content-Type. # # 3. header - future default, do not inspect the path for an extension and # use only the Accept header to determine the return Content-Type. policy = config.content_negotiation_policy # TODO - sometimes the client sends > 1 accept header value with ','. accept = str(request.accept).split(',')[0] valid_accept = accept and not ('application/*' in accept or 'text/*' in accept or '*/*' in accept) new_path: Optional[str] = path path_no_extension, extension = _d(new_path, True) accept_from_extension = accept if extension: accept_from_extension = _ctypes.get(extension, accept) if policy == 'extension': new_path = path_no_extension if not valid_accept: accept = accept_from_extension elif policy == 'adaptive': if not valid_accept: new_path = path_no_extension accept = accept_from_extension if not accept: log.warning('Could not determine accepted response type') raise exc.exception_response(400) q: Optional[str] if pfx and new_path: q = f'{{{pfx}}}{new_path}' new_path = f'/{alias}/{new_path}' else: q = new_path try: accepter = MediaAccept(accept) for p in request.registry.plumbings: state = { entry: True, 'headers': { 'Content-Type': None }, 'accept': accepter, 'url': request.current_route_url(), 'select': q, 'match': match.lower() if match else match, 'path': new_path, 'stats': {}, } r = p.process(request.registry.md, state=state, raise_exceptions=True, scheduler=request.registry.scheduler) log.debug(f'Plumbing process result: {r}') if r is None: r = [] response = Response() _headers = state.get('headers', {}) response.headers.update(_headers) ctype = _headers.get('Content-Type', None) if not ctype: r, t = _fmt(r, accepter) ctype = t response.text = b2u(r) response.size = len(r) response.content_type = ctype cache_ttl = int(state.get('cache', 0)) response.expires = datetime.now() + timedelta(seconds=cache_ttl) return response except ResourceException as ex: import traceback log.debug(traceback.format_exc()) log.warning(f'Exception from processing pipeline: {ex}') raise exc.exception_response(409) except BaseException as ex: import traceback log.debug(traceback.format_exc()) log.error(f'Exception from processing pipeline: {ex}') raise exc.exception_response(500) if request.method == 'GET': raise exc.exception_response(404)
def attributes(self): return b2u(list(self._attributes()))
def _attributes(self): ix = self.storage.open_index() with ix.reader() as reader: for n in reader.indexed_field_names(): if n in ATTRS: yield b2u(ATTRS[n])
def collections(self): return [b2u(ref) for ref in self.parts.keys()]
def _unpickle(self, x): return json.loads(b2u(x))
def test_b2u(self): assert int(b2u(b'1')) == 1 assert b2u('kaka') == 'kaka'
def entitiesdescriptor( entities, name, lookup_fn=None, cache_duration=None, valid_until=None, validate=True, filter_invalid=True, copy=True, nsmap=None, ): """ :param lookup_fn: a function used to lookup entities by name - set to None to skip resolving :param entities: a set of entities specifiers (lookup is used to find entities from this set) :param name: the @Name attribute :param cache_duration: an XML timedelta expression, eg PT1H for 1hr :param valid_until: a relative time eg 2w 4d 1h for 2 weeks, 4 days and 1hour from now. :param copy: set to False to avoid making a copy of all the entities in list. This may be dangerous. :param validate: set to False to skip schema validation of the resulting EntitiesDescriptor element. This is dangerous! :param filter_invalid: remove invalid EntitiesDescriptor elements from aggregate :param nsmap: additional namespace definitions to include in top level EntitiesDescriptor element Produce an EntityDescriptors set from a list of entities. Optional Name, cacheDuration and validUntil are affixed. """ if nsmap is None: nsmap = dict() nsmap.update(NS) if lookup_fn is not None: entities = resolve_entities(entities, lookup_fn=lookup_fn) for entity in entities: nsmap.update(entity.nsmap) log.debug("selecting %d entities before validation" % len(entities)) attrs = dict(Name=name, nsmap=nsmap) if cache_duration is not None: attrs['cacheDuration'] = cache_duration if valid_until is not None: attrs['validUntil'] = valid_until t = etree.Element("{%s}EntitiesDescriptor" % NS['md'], **attrs) for entity in entities: ent_insert = entity if copy: ent_insert = deepcopy(ent_insert) t.append(ent_insert) if config.devel_write_xml_to_file: import os with open("/tmp/pyff_entities_out-{}.xml".format(os.getpid()), "w") as fd: fd.write(b2u(dumptree(t))) if validate: validation_errors = dict() t = filter_or_validate(t, filter_invalid=filter_invalid, base_url=name, source="request", validation_errors=validation_errors) for base_url, err in validation_errors.items(): log.error("Validation error: @ {}: {}".format(base_url, err)) return t