def lookup(self, key):
log.debug("redis store lookup: %s" % key)
if '+' in key:
hk = hex_digest(key)
if not self.rc.exists("%s#members" % hk):
self.rc.zinterstore("%s#members" % hk, ["%s#members" % k for k in key.split('+')], 'min')
self.rc.expire("%s#members" % hk, 30) # XXX bad juju - only to keep clients from hammering
return self.lookup(hk)
m = re.match("^(.+)=(.+)$", key)
if m:
return self.lookup("{%s}%s" % (m.group(1), m.group(2)))
m = re.match("^{(.+)}(.+)$", key)
if m and ';' in m.group(2):
hk = hex_digest(key)
if not self.rc.exists("%s#members" % hk):
self.rc.zunionstore("%s#members" % hk,
["{%s}%s#members" % (m.group(1), v) for v in m.group(2).split(';')], 'min')
self.rc.expire("%s#members" % hk, 30) # XXX bad juju - only to keep clients from hammering
return self.lookup(hk)
elif self.rc.exists("%s#alias" % key):
return self.lookup(self.rc.get("%s#alias" % key))
elif self.rc.exists("%s#metadata" % key):
return [self._get_metadata(key)]
else:
return self._members(key)
def lookup(self, key):
#log.debug("redis store lookup: %s" % key)
if '+' in key:
hk = hex_digest(key)
if not self.rc.exists("%s#members" % hk):
self.rc.zinterstore("%s#members" % hk,
["%s#members" % k for k in key.split('+')],
'min')
self.rc.expire(
"%s#members" % hk,
30) # XXX bad juju - only to keep clients from hammering
return self.lookup(hk)
m = re.match("^(.+)=(.+)$", key)
if m:
return self.lookup("{%s}%s" % (m.group(1), m.group(2)))
m = re.match("^{(.+)}(.+)$", key)
if m and ';' in m.group(2):
hk = hex_digest(key)
if not self.rc.exists("%s#members" % hk):
self.rc.zunionstore("%s#members" % hk, [
"{%s}%s#members" % (m.group(1), v)
for v in m.group(2).split(';')
], 'min')
self.rc.expire(
"%s#members" % hk,
30) # XXX bad juju - only to keep clients from hammering
return self.lookup(hk)
elif self.rc.exists("%s#alias" % key):
return self.lookup(self.rc.get("%s#alias" % key))
elif self.rc.exists("%s#metadata" % key):
return [self._get_metadata(key)]
else:
return self._members(key)
def update(self,
t,
tid=None,
ts=None,
merge_strategy=None): # TODO: merge ?
log.debug("redis store update: %s: %s" % (t, tid))
relt = root(t)
ne = 0
if ts is None:
ts = int(
_now() +
3600 * 24 * 4) # 4 days is the arbitrary default expiration
if relt.tag == "{%s}EntityDescriptor" % NS['md']:
if tid is None:
tid = relt.get('entityID')
with self.rc.pipeline() as p:
self.update_entity(relt, t, tid, ts, p)
entity_id = relt.get("entityID")
if entity_id is not None:
self.membership("entities", entity_id, ts, p)
for ea, eav in entity_attribute_dict(relt).iteritems():
for v in eav:
# log.debug("%s=%s" % (ea, v))
self.membership("{%s}%s" % (ea, v), tid, ts, p)
p.zadd("%s#values" % ea, v, ts)
p.sadd("#attributes", ea)
for hn in ('sha1', 'sha256', 'md5'):
tid_hash = hex_digest(tid, hn)
p.set("{%s}%s#alias" % (hn, tid_hash), tid)
if ts is not None:
p.expireat(tid_hash, ts)
p.execute()
ne += 1
elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']:
if tid is None:
tid = relt.get('Name')
ts = self._expiration(relt)
with self.rc.pipeline() as p:
self.update_entity(relt, t, tid, ts, p)
for e in iter_entities(t):
ne += self.update(e, ts=ts)
entity_id = e.get("entityID")
if entity_id is not None:
self.membership(tid, entity_id, ts, p)
self.membership("entities", entity_id, ts, p)
p.execute()
else:
raise ValueError("Bad metadata top-level element: '%s'" %
root(t).tag)
return ne
def update(self, t, tid=None, ts=None, merge_strategy=None): # TODO: merge ?
log.debug("redis store update: %s: %s" % (t, tid))
relt = root(t)
ne = 0
if ts is None:
ts = int(_now() + 3600 * 24 * 4) # 4 days is the arbitrary default expiration
if relt.tag == "{%s}EntityDescriptor" % NS['md']:
if tid is None:
tid = relt.get('entityID')
with self.rc.pipeline() as p:
self.update_entity(relt, t, tid, ts, p)
entity_id = relt.get("entityID")
if entity_id is not None:
self.membership("entities", entity_id, ts, p)
for ea, eav in entity_attribute_dict(relt).iteritems():
for v in eav:
# log.debug("%s=%s" % (ea, v))
self.membership("{%s}%s" % (ea, v), tid, ts, p)
p.zadd("%s#values" % ea, v, ts)
p.sadd("#attributes", ea)
for hn in ('sha1', 'sha256', 'md5'):
tid_hash = hex_digest(tid, hn)
p.set("{%s}%s#alias" % (hn, tid_hash), tid)
if ts is not None:
p.expireat(tid_hash, ts)
p.execute()
ne += 1
elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']:
if tid is None:
tid = relt.get('Name')
ts = self._expiration(relt)
with self.rc.pipeline() as p:
self.update_entity(relt, t, tid, ts, p)
for e in iter_entities(t):
ne += self.update(e, ts=ts)
entity_id = e.get("entityID")
if entity_id is not None:
self.membership(tid, entity_id, ts, p)
self.membership("entities", entity_id, ts, p)
p.execute()
else:
raise ValueError("Bad metadata top-level element: '%s'" % root(t).tag)
return ne
def update(self, t, tid=None, etag=None, lazy=True):
relt = root(t)
assert relt is not None
if relt.tag == "{%s}EntityDescriptor" % NS['md']:
ref = object_id(relt)
parts = None
if ref in self.parts:
parts = self.parts[ref]
if etag is not None and (parts is None
or parts.get('etag', None) != etag):
self.parts[ref] = {
'id': relt.get('entityID'),
'etag': etag,
'count': 1,
'items': [ref]
}
self.objects[ref] = relt
self._last_modified = datetime.now()
elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']:
if tid is None:
tid = relt.get('Name')
if etag is None:
etag = hex_digest(dumptree(t, pretty_print=False), 'sha256')
parts = None
if tid in self.parts:
parts = self.parts[tid]
if parts is None or parts.get('etag', None) != etag:
items = set()
for e in iter_entities(t):
ref = object_id(e)
items.add(ref)
self.objects[ref] = e
self.parts[tid] = {
'id': tid,
'count': len(items),
'etag': etag,
'items': list(items)
}
self._last_modified = datetime.now()
if not lazy:
self._reindex()
def load_resource(
self, getter: Callable[[str], Response]
) -> Tuple[Optional[str], int, ResourceInfo]:
data: Optional[str] = None
status: int = 500
info = self.add_info()
log.debug("Loading resource {}".format(self.url))
if not self.url:
log.error(f'No URL for resource {self}')
return data, status, info
try:
r = getter(self.url)
info.http_headers = dict(r.headers)
log.debug(
"got status_code={:d}, encoding={} from_cache={} from {}".
format(r.status_code, r.encoding,
getattr(r, "from_cache", False), self.url))
status = r.status_code
info.reason = r.reason
if r.ok:
data = r.text
_etag = r.headers.get('ETag', None)
if not _etag:
_etag = hex_digest(r.text, 'sha256')
self.etag = _etag
elif self.local_copy_fn is not None:
log.warning(
"Got status={:d} while getting {}. Attempting fallback to local copy."
.format(r.status_code, self.url))
data = self.load_backup()
if data is not None and len(data) > 0:
info.reason = "Retrieved from local cache because status: {} != 200".format(
status)
status = 218
info.status_code = str(status)
except IOError as ex:
if self.local_copy_fn is not None:
log.warning(
"caught exception from {} - trying local backup: {}".
format(self.url, ex))
data = self.load_backup()
if data is not None and len(data) > 0:
info.reason = "Retrieved from local cache because exception: {}".format(
ex)
status = 218
if data is None or not len(data) > 0:
raise ex # propagate exception if we can't find a backup
if data is None or not len(data) > 0:
raise ResourceException("failed to fetch {} (status: {:d})".format(
self.url, status))
info.state = ResourceLoadState.Fetched
return data, status, info
