def lookup(self, key):
if key == 'entities' or key is None:
return self._entities()
bkey = six.b(key)
if bkey in self.objects:
return [self.objects.get(bkey)]
if bkey in self.parts:
res = []
part = self.parts.get(bkey)
for item in part['items']:
res.extend(self.lookup(item))
return res
key = self._prep_key(key)
qp = QueryParser("object_id", schema=self.schema)
q = qp.parse(key)
lst = set()
with self.index.searcher() as searcher:
results = searcher.search(q, limit=None)
for result in results:
e = self.objects.get(result['object_id'], None)
if e is not None:
lst.add(e)
return b2u(list(lst))
def attribute(self, a):
if a in ATTRS_INV:
n = ATTRS_INV[a]
ix = self.storage.open_index()
with ix.searcher() as searcher:
return b2u(list(searcher.lexicon(n)))
else:
return []
def _entities(self):
lst = set()
for ref_data in self.parts.values():
for ref in ref_data['items']:
e = self.objects.get(ref, None)
if e is not None:
lst.add(e)
return b2u(list(lst))
def _reindex(self):
log.debug("indexing the store...")
self._last_index_time = datetime.now()
seen = set()
refs = set([b2u(s) for s in self.objects.keys()])
parts = self.parts.values()
for ref in refs:
for part in parts:
if ref in part['items']:
seen.add(ref)
ix = self.storage.open_index()
lock = ix.lock("reindex")
try:
log.debug("waiting for index lock")
lock.acquire(True)
log.debug("got index lock")
with ix.writer() as writer:
for ref in refs:
if ref not in seen:
log.debug("removing unseen ref {}".format(ref))
del self.objects[ref]
del self.parts[ref]
log.debug("updating index")
for e in self.objects.values():
info = self._index_prep(entity_simple_info(e))
ref = object_id(e)
writer.add_document(object_id=ref, **info)
writer.mergetype = CLEAR
finally:
try:
log.debug("releasing index lock")
lock.release()
except ThreadError as ex:
pass
def test_b2u(self):
assert(int(b2u(b'1')) == 1)
assert(b2u('kaka') == 'kaka')
def process_handler(request: Request) -> Response:
"""
The main request handler for pyFF. Implements API call hooks and content negotiation.
:param request: the HTTP request object
:return: the data to send to the client
"""
_ctypes = {
'xml': 'application/samlmetadata+xml;application/xml;text/xml',
'json': 'application/json'
}
def _d(x: Optional[str],
do_split: bool = True) -> Tuple[Optional[str], Optional[str]]:
""" Split a path into a base component and an extension. """
if x is not None:
x = x.strip()
if x is None or len(x) == 0:
return None, None
if '.' in x:
(pth, dot, extn) = x.rpartition('.')
assert dot == '.'
if extn in _ctypes:
return pth, extn
return x, None
log.debug(f'Processing request: {request}')
if request.matchdict is None:
raise exc.exception_response(400)
if request.body:
try:
request.matchdict.update(request.json_body)
except ValueError as ex:
pass
entry = request.matchdict.get('entry', 'request')
path_elem = list(request.matchdict.get('path', []))
match = request.params.get('q', request.params.get('query', None))
# Enable matching on scope.
match = match.split(
'@').pop() if match and not match.endswith('@') else match
log.debug("match={}".format(match))
if not path_elem:
path_elem = ['entities']
alias = path_elem.pop(0)
path = '/'.join(path_elem)
# Ugly workaround bc WSGI drops double-slashes.
path = path.replace(':/', '://')
msg = "handling entry={}, alias={}, path={}"
log.debug(msg.format(entry, alias, path))
pfx = None
if 'entities' not in alias:
pfx = request.registry.aliases.get(alias, None)
if pfx is None:
raise exc.exception_response(404)
# content_negotiation_policy is one of three values:
# 1. extension - current default, inspect the path and if it ends in
# an extension, e.g. .xml or .json, always strip off the extension to
# get the entityID and if no accept header or a wildcard header, then
# use the extension to determine the return Content-Type.
#
# 2. adaptive - only if no accept header or if a wildcard, then inspect
# the path and if it ends in an extension strip off the extension to
# get the entityID and use the extension to determine the return
# Content-Type.
#
# 3. header - future default, do not inspect the path for an extension and
# use only the Accept header to determine the return Content-Type.
policy = config.content_negotiation_policy
# TODO - sometimes the client sends > 1 accept header value with ','.
accept = str(request.accept).split(',')[0]
valid_accept = accept and not ('application/*' in accept
or 'text/*' in accept or '*/*' in accept)
new_path: Optional[str] = path
path_no_extension, extension = _d(new_path, True)
accept_from_extension = accept
if extension:
accept_from_extension = _ctypes.get(extension, accept)
if policy == 'extension':
new_path = path_no_extension
if not valid_accept:
accept = accept_from_extension
elif policy == 'adaptive':
if not valid_accept:
new_path = path_no_extension
accept = accept_from_extension
if not accept:
log.warning('Could not determine accepted response type')
raise exc.exception_response(400)
q: Optional[str]
if pfx and new_path:
q = f'{{{pfx}}}{new_path}'
new_path = f'/{alias}/{new_path}'
else:
q = new_path
try:
accepter = MediaAccept(accept)
for p in request.registry.plumbings:
state = {
entry: True,
'headers': {
'Content-Type': None
},
'accept': accepter,
'url': request.current_route_url(),
'select': q,
'match': match.lower() if match else match,
'path': new_path,
'stats': {},
}
r = p.process(request.registry.md,
state=state,
raise_exceptions=True,
scheduler=request.registry.scheduler)
log.debug(f'Plumbing process result: {r}')
if r is None:
r = []
response = Response()
_headers = state.get('headers', {})
response.headers.update(_headers)
ctype = _headers.get('Content-Type', None)
if not ctype:
r, t = _fmt(r, accepter)
ctype = t
response.text = b2u(r)
response.size = len(r)
response.content_type = ctype
cache_ttl = int(state.get('cache', 0))
response.expires = datetime.now() + timedelta(seconds=cache_ttl)
return response
except ResourceException as ex:
import traceback
log.debug(traceback.format_exc())
log.warning(f'Exception from processing pipeline: {ex}')
raise exc.exception_response(409)
except BaseException as ex:
import traceback
log.debug(traceback.format_exc())
log.error(f'Exception from processing pipeline: {ex}')
raise exc.exception_response(500)
if request.method == 'GET':
raise exc.exception_response(404)
def attributes(self):
return b2u(list(self._attributes()))
def _attributes(self):
ix = self.storage.open_index()
with ix.reader() as reader:
for n in reader.indexed_field_names():
if n in ATTRS:
yield b2u(ATTRS[n])
def collections(self):
return [b2u(ref) for ref in self.parts.keys()]
def _unpickle(self, x):
return json.loads(b2u(x))
def test_b2u(self):
assert int(b2u(b'1')) == 1
assert b2u('kaka') == 'kaka'
def entitiesdescriptor(
entities,
name,
lookup_fn=None,
cache_duration=None,
valid_until=None,
validate=True,
filter_invalid=True,
copy=True,
nsmap=None,
):
"""
:param lookup_fn: a function used to lookup entities by name - set to None to skip resolving
:param entities: a set of entities specifiers (lookup is used to find entities from this set)
:param name: the @Name attribute
:param cache_duration: an XML timedelta expression, eg PT1H for 1hr
:param valid_until: a relative time eg 2w 4d 1h for 2 weeks, 4 days and 1hour from now.
:param copy: set to False to avoid making a copy of all the entities in list. This may be dangerous.
:param validate: set to False to skip schema validation of the resulting EntitiesDescriptor element. This is dangerous!
:param filter_invalid: remove invalid EntitiesDescriptor elements from aggregate
:param nsmap: additional namespace definitions to include in top level EntitiesDescriptor element
Produce an EntityDescriptors set from a list of entities. Optional Name, cacheDuration and validUntil are affixed.
"""
if nsmap is None:
nsmap = dict()
nsmap.update(NS)
if lookup_fn is not None:
entities = resolve_entities(entities, lookup_fn=lookup_fn)
for entity in entities:
nsmap.update(entity.nsmap)
log.debug("selecting %d entities before validation" % len(entities))
attrs = dict(Name=name, nsmap=nsmap)
if cache_duration is not None:
attrs['cacheDuration'] = cache_duration
if valid_until is not None:
attrs['validUntil'] = valid_until
t = etree.Element("{%s}EntitiesDescriptor" % NS['md'], **attrs)
for entity in entities:
ent_insert = entity
if copy:
ent_insert = deepcopy(ent_insert)
t.append(ent_insert)
if config.devel_write_xml_to_file:
import os
with open("/tmp/pyff_entities_out-{}.xml".format(os.getpid()),
"w") as fd:
fd.write(b2u(dumptree(t)))
if validate:
validation_errors = dict()
t = filter_or_validate(t,
filter_invalid=filter_invalid,
base_url=name,
source="request",
validation_errors=validation_errors)
for base_url, err in validation_errors.items():
log.error("Validation error: @ {}: {}".format(base_url, err))
return t