def lookup(self, key): log.debug("redis store lookup: %s" % key) if '+' in key: hk = hex_digest(key) if not self.rc.exists("%s#members" % hk): self.rc.zinterstore("%s#members" % hk, ["%s#members" % k for k in key.split('+')], 'min') self.rc.expire("%s#members" % hk, 30) # XXX bad juju - only to keep clients from hammering return self.lookup(hk) m = re.match("^(.+)=(.+)$", key) if m: return self.lookup("{%s}%s" % (m.group(1), m.group(2))) m = re.match("^{(.+)}(.+)$", key) if m and ';' in m.group(2): hk = hex_digest(key) if not self.rc.exists("%s#members" % hk): self.rc.zunionstore("%s#members" % hk, ["{%s}%s#members" % (m.group(1), v) for v in m.group(2).split(';')], 'min') self.rc.expire("%s#members" % hk, 30) # XXX bad juju - only to keep clients from hammering return self.lookup(hk) elif self.rc.exists("%s#alias" % key): return self.lookup(self.rc.get("%s#alias" % key)) elif self.rc.exists("%s#metadata" % key): return [self._get_metadata(key)] else: return self._members(key)
def lookup(self, key): #log.debug("redis store lookup: %s" % key) if '+' in key: hk = hex_digest(key) if not self.rc.exists("%s#members" % hk): self.rc.zinterstore("%s#members" % hk, ["%s#members" % k for k in key.split('+')], 'min') self.rc.expire( "%s#members" % hk, 30) # XXX bad juju - only to keep clients from hammering return self.lookup(hk) m = re.match("^(.+)=(.+)$", key) if m: return self.lookup("{%s}%s" % (m.group(1), m.group(2))) m = re.match("^{(.+)}(.+)$", key) if m and ';' in m.group(2): hk = hex_digest(key) if not self.rc.exists("%s#members" % hk): self.rc.zunionstore("%s#members" % hk, [ "{%s}%s#members" % (m.group(1), v) for v in m.group(2).split(';') ], 'min') self.rc.expire( "%s#members" % hk, 30) # XXX bad juju - only to keep clients from hammering return self.lookup(hk) elif self.rc.exists("%s#alias" % key): return self.lookup(self.rc.get("%s#alias" % key)) elif self.rc.exists("%s#metadata" % key): return [self._get_metadata(key)] else: return self._members(key)
def update(self, t, tid=None, ts=None, merge_strategy=None): # TODO: merge ? log.debug("redis store update: %s: %s" % (t, tid)) relt = root(t) ne = 0 if ts is None: ts = int( _now() + 3600 * 24 * 4) # 4 days is the arbitrary default expiration if relt.tag == "{%s}EntityDescriptor" % NS['md']: if tid is None: tid = relt.get('entityID') with self.rc.pipeline() as p: self.update_entity(relt, t, tid, ts, p) entity_id = relt.get("entityID") if entity_id is not None: self.membership("entities", entity_id, ts, p) for ea, eav in entity_attribute_dict(relt).iteritems(): for v in eav: # log.debug("%s=%s" % (ea, v)) self.membership("{%s}%s" % (ea, v), tid, ts, p) p.zadd("%s#values" % ea, v, ts) p.sadd("#attributes", ea) for hn in ('sha1', 'sha256', 'md5'): tid_hash = hex_digest(tid, hn) p.set("{%s}%s#alias" % (hn, tid_hash), tid) if ts is not None: p.expireat(tid_hash, ts) p.execute() ne += 1 elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']: if tid is None: tid = relt.get('Name') ts = self._expiration(relt) with self.rc.pipeline() as p: self.update_entity(relt, t, tid, ts, p) for e in iter_entities(t): ne += self.update(e, ts=ts) entity_id = e.get("entityID") if entity_id is not None: self.membership(tid, entity_id, ts, p) self.membership("entities", entity_id, ts, p) p.execute() else: raise ValueError("Bad metadata top-level element: '%s'" % root(t).tag) return ne
def update(self, t, tid=None, ts=None, merge_strategy=None): # TODO: merge ? log.debug("redis store update: %s: %s" % (t, tid)) relt = root(t) ne = 0 if ts is None: ts = int(_now() + 3600 * 24 * 4) # 4 days is the arbitrary default expiration if relt.tag == "{%s}EntityDescriptor" % NS['md']: if tid is None: tid = relt.get('entityID') with self.rc.pipeline() as p: self.update_entity(relt, t, tid, ts, p) entity_id = relt.get("entityID") if entity_id is not None: self.membership("entities", entity_id, ts, p) for ea, eav in entity_attribute_dict(relt).iteritems(): for v in eav: # log.debug("%s=%s" % (ea, v)) self.membership("{%s}%s" % (ea, v), tid, ts, p) p.zadd("%s#values" % ea, v, ts) p.sadd("#attributes", ea) for hn in ('sha1', 'sha256', 'md5'): tid_hash = hex_digest(tid, hn) p.set("{%s}%s#alias" % (hn, tid_hash), tid) if ts is not None: p.expireat(tid_hash, ts) p.execute() ne += 1 elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']: if tid is None: tid = relt.get('Name') ts = self._expiration(relt) with self.rc.pipeline() as p: self.update_entity(relt, t, tid, ts, p) for e in iter_entities(t): ne += self.update(e, ts=ts) entity_id = e.get("entityID") if entity_id is not None: self.membership(tid, entity_id, ts, p) self.membership("entities", entity_id, ts, p) p.execute() else: raise ValueError("Bad metadata top-level element: '%s'" % root(t).tag) return ne
def update(self, t, tid=None, etag=None, lazy=True): relt = root(t) assert relt is not None if relt.tag == "{%s}EntityDescriptor" % NS['md']: ref = object_id(relt) parts = None if ref in self.parts: parts = self.parts[ref] if etag is not None and (parts is None or parts.get('etag', None) != etag): self.parts[ref] = { 'id': relt.get('entityID'), 'etag': etag, 'count': 1, 'items': [ref] } self.objects[ref] = relt self._last_modified = datetime.now() elif relt.tag == "{%s}EntitiesDescriptor" % NS['md']: if tid is None: tid = relt.get('Name') if etag is None: etag = hex_digest(dumptree(t, pretty_print=False), 'sha256') parts = None if tid in self.parts: parts = self.parts[tid] if parts is None or parts.get('etag', None) != etag: items = set() for e in iter_entities(t): ref = object_id(e) items.add(ref) self.objects[ref] = e self.parts[tid] = { 'id': tid, 'count': len(items), 'etag': etag, 'items': list(items) } self._last_modified = datetime.now() if not lazy: self._reindex()
def load_resource( self, getter: Callable[[str], Response] ) -> Tuple[Optional[str], int, ResourceInfo]: data: Optional[str] = None status: int = 500 info = self.add_info() log.debug("Loading resource {}".format(self.url)) if not self.url: log.error(f'No URL for resource {self}') return data, status, info try: r = getter(self.url) info.http_headers = dict(r.headers) log.debug( "got status_code={:d}, encoding={} from_cache={} from {}". format(r.status_code, r.encoding, getattr(r, "from_cache", False), self.url)) status = r.status_code info.reason = r.reason if r.ok: data = r.text _etag = r.headers.get('ETag', None) if not _etag: _etag = hex_digest(r.text, 'sha256') self.etag = _etag elif self.local_copy_fn is not None: log.warning( "Got status={:d} while getting {}. Attempting fallback to local copy." .format(r.status_code, self.url)) data = self.load_backup() if data is not None and len(data) > 0: info.reason = "Retrieved from local cache because status: {} != 200".format( status) status = 218 info.status_code = str(status) except IOError as ex: if self.local_copy_fn is not None: log.warning( "caught exception from {} - trying local backup: {}". format(self.url, ex)) data = self.load_backup() if data is not None and len(data) > 0: info.reason = "Retrieved from local cache because exception: {}".format( ex) status = 218 if data is None or not len(data) > 0: raise ex # propagate exception if we can't find a backup if data is None or not len(data) > 0: raise ResourceException("failed to fetch {} (status: {:d})".format( self.url, status)) info.state = ResourceLoadState.Fetched return data, status, info