class CoapLRUCache(Cache): def __init__(self, mode, max_dim): """ Initialise an LRU cache for the Coap. :param max_dim: max number of elements in the cache :param mode: used to differentiate between a cache used in a forward-proxy or in a reverse-proxy """ Cache.__init__(self, mode, max_dim) self.cache = LRUCache(maxsize=max_dim) def __str__(self): msg = [] for e in list(self.cache.values()): msg.append(str(e)) return ("Cache Size: {sz}\n" + "\n".join(msg)) def debug_print(self): """ :return: a debug printout for the current cache. """ return ("size = %s\n%s" % ( self.cache.currsize, '\n'.join([ ( "element.max age %s\n"\ "element.uri %s\n"\ "element.freshness %s" ) % ( element.max_age, element.uri, element.freshness ) for key, element in list(self.cache.items()) ])))
def fixLinks(self, linkTargetSet): import gc from cachetools import LRUCache gc.collect() dirn = self._filename filenameList = self._filenameList fileByWord = {} for line in open(join(dirn, "index.txt"), encoding="utf-8"): line = line.rstrip("\n") if not line: continue entryIndex, wordEsc, filename, _ = line.split("\t") entryIndex = int(entryIndex) # entryId = f"entry{entryIndex}" word = unescapeNTB(wordEsc) if word not in linkTargetSet: continue if word in fileByWord: # log.info(f'fileByWord[{word}]={fileByWord[word]}, filename={filename}') fileByWord[word].append((filename, entryIndex)) else: fileByWord[word] = [(filename, entryIndex)] linksByFile = LRUCache(maxsize=100) # with open(join(dirn, "fileByWord.json"), "w") as fileByWordFile: # json.dump(fileByWord, fileByWordFile, ensure_ascii=False, indent="\t") def getLinksByFile(fileIndex): _file = linksByFile.get(fileIndex) if _file is not None: return _file _file = open( join(dirn, f"links{fileIndex}"), mode="a", encoding="utf-8", ) linksByFile[fileIndex] = _file return _file log.info("") for line in open(join(dirn, "links.txt"), encoding="utf-8"): line = line.rstrip("\n") if not line: continue target, fileIndex, x_start, x_size = line.split("\t") target = unescapeNTB(target) if target not in fileByWord: targetNew = "" else: targetFilename, targetEntryIndex = fileByWord[target][0] if targetFilename == filename: continue targetNew = f"{targetFilename}#entry{targetEntryIndex}" _file = getLinksByFile(int(fileIndex)) _file.write(f"{x_start}\t{x_size}\t{targetNew}\n") _file.flush() for _, _file in linksByFile.items(): _file.close() del linksByFile linkTargetSet.clear() del fileByWord, linkTargetSet gc.collect() entry_url_fmt = self._glos.getInfo("entry_url") re_href = re.compile( b' href="[^<>"]*?"', re.I, ) for fileIndex, filename in enumerate(filenameList): if not isfile(join(dirn, f"links{fileIndex}")): continue with open(join(dirn, filename), mode="rb") as inFile: with open(join(dirn, f"{filename}.new"), mode="wb") as outFile: for linkLine in open(join(dirn, f"links{fileIndex}"), "rb"): outFile.flush() linkLine = linkLine.rstrip(b"\n") x_start, x_size, target = linkLine.split(b"\t") outFile.write( inFile.read(int(x_start, 16) - inFile.tell())) curLink = inFile.read(int(x_size, 16)) if target: outFile.write( re_href.sub( b' href="./' + target + b'"', curLink, )) continue if not entry_url_fmt: outFile.write( curLink.replace( b' href="#', b' class="broken" href="#', )) continue _st = curLink.decode("utf-8") i = _st.find('href="#') j = _st.find('"', i + 7) word = _st[i + 7:j] url = entry_url_fmt.format(word=word) outFile.write( (_st[:i] + f'class="broken" href="{url}"' + _st[j + 1:]).encode("utf-8")) outFile.write(inFile.read()) os.rename(join(dirn, f"{filename}.new"), join(dirn, filename)) os.remove(join(dirn, f"links{fileIndex}"))
class Server(Component): def init(self, args, db, hosts, logger): self.args = args self.db = db self.hosts = hosts self.logger = logger self.bind = args.bind self.forward = args.forward self.peers = {} self.requests = {} self.cache = LRUCache(maxsize=args.cachesize) if args.daemon: Daemon(args.pidfile).register(self) if args.debug: Debugger(events=args.verbose, logger=logger).register(self) self.transport = UDPServer(self.bind).register(self) self.protocol = DNS().register(self) def ready(self, server, bind): self.logger.info( "DNS Server Ready! Listening on {0:s}:{1:d}".format(*bind)) Timer(1, Event.create("ttl"), persist=True).register(self) def ttl(self): for k, rrs in self.cache.items()[:]: if any(rr.ttl == 0 for rr in rrs): qname, qtype, qclass = k self.logger.info("Expired Entry: {0:s} {1:s} {2:s}".format( CLASS.get(qclass), QTYPE.get(qtype), qname)) del self.cache[k] else: for rr in rrs: rr.ttl -= 1 def request(self, peer, request): qname = str(request.q.qname) qtype = request.q.qtype qclass = request.q.qclass key = (qname, qtype, qclass) if key in self.cache: self.logger.info( "Cached Request ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname)) reply = request.reply() for rr in self.cache[key]: reply.add_answer(rr) self.fire(write(peer, reply.pack())) return if key in self.hosts: self.logger.info( "Local Hosts Request ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname)) rr = [RR(qname, rdata=A(self.hosts[key]))] reply = request.reply() reply.add_answer(*rr) self.cache[key] = rr self.fire(write(peer, reply.pack())) return records = Record.objects.filter(rname=qname, rclass=qclass, rtype=qtype) if records: self.logger.info( "Authoritative Request ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname)) rr = [record.rr for record in records] reply = request.reply() reply.add_answer(*rr) self.cache[key] = rr self.fire(write(peer, reply.pack())) return self.logger.info("Request ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname)) lookup = DNSRecord(q=DNSQuestion(qname, qtype, qclass)) id = lookup.header.id self.peers[id] = peer self.requests[id] = request self.fire(write((self.forward, 53), lookup.pack())) def response(self, peer, response): id = response.header.id qname = str(response.q.qname) qtype = response.q.qtype qclass = response.q.qclass if id not in self.peers: self.logger.info( "Unknown Response ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname)) return peer = self.peers[id] request = self.requests[id] key = (str(request.q.qname), request.q.qtype, request.q.qclass) reply = request.reply() reply.add_answer(*response.rr) self.cache[key] = reply.rr self.fire(write(peer, reply.pack())) del self.peers[id] del self.requests[id]
class CoapLRUCache(CoapCache): def __init__(self, max_dim): """ :param max_dim: """ self.cache = LRUCache(maxsize=max_dim) def update(self, key, element): """ :param key: :param element: :return: """ logger.debug("updating cache, key: %s, element: %s", \ key.hashkey, element) self.cache.update([(key.hashkey, element)]) def get(self, key): """ :param key: :return: CacheElement """ try: response = self.cache[key.hashkey] except KeyError: logger.debug("problem here", exc_info=1) response = None return response def is_full(self): """ :return: """ if self.cache.currsize == self.cache.maxsize: return True return False def is_empty(self): """ :return: """ if self.cache.currsize == 0: return True return False def __str__(self): msg = [] for e in self.cache.values(): msg.append(str(e)) return "Cache Size: {sz}\n" + "\n".join(msg) def debug_print(self): """ :return: """ return ("size = %s\n%s" % ( self.cache.currsize, '\n'.join([ ( "element.max age %s\n"\ "element.uri %s\n"\ "element.freshness %s" ) % ( element.max_age, element.uri, element.freshness ) for key, element in list(self.cache.items()) ])))
class Server(Component): channel = "server" def init(self, args, db, hosts, logger): self.args = args self.db = db self.hosts = hosts self.logger = logger self.bind = args.bind self.forward = args.forward self.peers = {} self.requests = {} self.cache = LRUCache(maxsize=args.cachesize) if args.daemon: Daemon(args.pidfile).register(self) if args.debug: Debugger(events=args.verbose, logger=logger).register(self) self.transport = UDPServer( self.bind, channel=self.channel ).register(self) self.protocol = DNS(channel=self.channel).register(self) def ready(self, server, bind): self.logger.info( "DNS Server Ready! Listening on {0:s}:{1:d}".format(*bind) ) # Timer(1, Event.create("ttl"), persist=True, channel=self.channel).register(self) def ttl(self): for k, rrs in self.cache.items()[:]: if any(rr.ttl == 0 for rr in rrs): qname, qtype, qclass = k self.logger.info( "Expired Entry: {0:s} {1:s} {2:s}".format( CLASS.get(qclass), QTYPE.get(qtype), qname ) ) del self.cache[k] else: for rr in rrs: rr.ttl -= 1 def request(self, peer, request): qname = str(request.q.qname) qtype = request.q.qtype qclass = request.q.qclass key = (qname, qtype, qclass) if key in self.cache: self.logger.info( "Cached Request ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname ) ) reply = request.reply() for rr in self.cache[key]: reply.add_answer(rr) self.fire(write(peer, reply.pack())) return if key in self.hosts: self.logger.info( "Local Hosts Request ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname ) ) reply = request.reply() for rdata in self.hosts[key]: rr = RR( qname, rclass=CLASS.IN, rtype=QTYPE.AAAA if ":" in rdata else QTYPE.A, rdata=AAAA(rdata) if ":" in rdata else A(rdata) ) reply.add_answer(rr) self.cache[key] = rr self.fire(write(peer, reply.pack())) return records = Record.objects.filter(rname=qname) if not records: self.logger.info( "Request ({0:s}): {1:s} {2:s} {3:s} -> {4:s}:{5:d}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname, self.forward, 53 ) ) lookup = DNSRecord(q=DNSQuestion(qname, qtype, qclass)) id = lookup.header.id self.peers[id] = peer self.requests[id] = request self.fire(write((self.forward, 53), lookup.pack())) return self.logger.info( "Authoritative Request ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname ) ) rr = [] reply = request.reply() if len(records) == 1 and records[0].rtype == CNAME: rr.append(records[0].rr) records = Record.objects.filter(rname=records[0].rdata) for record in records: rr.append(record.rr) reply.add_answer(*rr) self.cache[key] = rr self.fire(write(peer, reply.pack())) def response(self, peer, response): id = response.header.id qname = str(response.q.qname) qtype = response.q.qtype qclass = response.q.qclass if id not in self.peers: self.logger.info( "Unknown Response ({0:s}): {1:s} {2:s} {3:s}".format( "{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype), qname ) ) return peer = self.peers[id] request = self.requests[id] key = (str(request.q.qname), request.q.qtype, request.q.qclass) reply = request.reply() reply.add_answer(*response.rr) self.cache[key] = reply.rr self.fire(write(peer, reply.pack())) del self.peers[id] del self.requests[id]
class CoapLRUCache(CoapCache): def __init__(self, max_dim): """ :param max_dim: """ self.cache = LRUCache(maxsize=max_dim) def update(self, key, element): """ :param key: :param element: :return: """ six.print_("updating cache") six.print_("key: ", key.hashkey) six.print_("element: ", element) self.cache.update([(key.hashkey, element)]) def get(self, key): """ :param key: :return: CacheElement """ try: response = self.cache[key.hashkey] except KeyError: six.print_("problem here") response = None return response def is_full(self): """ :return: """ if self.cache.currsize == self.cache.maxsize: return True return False def is_empty(self): """ :return: """ if self.cache.currsize == 0: return True return False def debug_print(self): """ :return: """ six.print_("size = ", self.cache.currsize) list = self.cache.items() for key, element in list: six.print_("element.max age ", element.max_age) six.print_("element.uri", element.uri) six.print_("element.freshness ", element.freshness)
class Cache(): def __init__(self,cachefilename,CACHE_SIZE,logger=logger('detoursCache.log')): self.lock = threading.RLock() self.cachefilename = cachefilename self.entry = LRUCache(maxsize=CACHE_SIZE) self.logger=logger self.hitcount=0 def hit(self): self.lock.acquire(blocking=1) try: self.hitcount+=1 finally: self.lock.release() def reset(self): self.lock.acquire(blocking=1) try: self.hitcount=0 finally: self.lock.release() def push(self,key,val): self.lock.acquire(blocking=1) try: self.entry[key]=val except: return finally: self.lock.release() def get(self,key): self.lock.acquire(blocking=1) try: return self.entry[key] except: return False finally: self.lock.release() def write_to_disk(self): self.lock.acquire(blocking=1) try: cachefile = open(self.cachefilename,'w') for key,val in self.entry.items(): print(key+'\t'+val,file=cachefile) cachefile.close() finally: self.lock.release() def load_from_disk(self): self.lock.acquire(blocking=1) try: if os.path.exists(self.cachefilename): with open(self.cachefilename, 'r') as f: for line in f: if line == "": continue rline = line.strip() splitvals=rline.split('\t') if len(splitvals) == 2: key=splitvals[0] valstr=splitvals[1] self.entry[key]=valstr else: continue except: self.logger.error("Failed to read existing cache file") raise("Error in loading previous cache file") finally: self.lock.release()
def fixLinks(self, linkTargetSet): import gc from cachetools import LRUCache gc.collect() dirn = self._filename filenameList = self._filenameList fileByWord = {} for line in open(join(dirn, "index.txt"), encoding="utf-8"): line = line.rstrip("\n") if not line: continue word, filename, _ = line.split("\t") word = unescapeNTB(word) if word not in linkTargetSet: continue fileByWord[word] = filename linksByFile = LRUCache(maxsize=100) def getLinksByFile(fileIndex): _file = linksByFile.get(fileIndex) if _file is not None: return _file _file = open( join(dirn, f"links{fileIndex}"), mode="a", encoding="utf-8", ) linksByFile[fileIndex] = _file return _file log.info("") for line in open(join(dirn, "links.txt"), encoding="utf-8"): line = line.rstrip("\n") if not line: continue target, fileIndex, x_start, x_size = line.split("\t") target = unescapeNTB(target) if target not in fileByWord: targetFilename = "" else: targetFilename = fileByWord[target] if targetFilename == filename: continue _file = getLinksByFile(int(fileIndex)) _file.write( f"{x_start}\t{x_size}\t{targetFilename}\n" ) _file.flush() for _, _file in linksByFile.items(): _file.close() del linksByFile linkTargetSet.clear() del fileByWord, linkTargetSet gc.collect() entry_url_fmt = self._glos.getInfo("entry_url") for fileIndex, filename in enumerate(filenameList): with open(join(dirn, filename), mode="rb") as inFile: with open(join(dirn, f"{filename}.new"), mode="wb") as outFile: for linkLine in open(join(dirn, f"links{fileIndex}"), "rb"): outFile.flush() linkLine = linkLine.rstrip(b"\n") x_start, x_size, targetFilename = linkLine.split(b"\t") outFile.write(inFile.read( int(x_start, 16) - inFile.tell() )) curLink = inFile.read(int(x_size, 16)) if targetFilename: outFile.write(curLink.replace( b' href="#', b' href="./' + targetFilename + b'#', )) continue if not entry_url_fmt: outFile.write(curLink.replace( b' href="#', b' class="broken" href="#', )) continue _st = curLink.decode("utf-8") i = _st.find('href="#') j = _st.find('"', i + 7) word = _st[i + 7:j] url = entry_url_fmt.format(word=word) outFile.write(( _st[:i] + f'class="broken" href="{url}"' + _st[j + 1:] ).encode("utf-8")) outFile.write(inFile.read()) os.rename(join(dirn, f"{filename}.new"), join(dirn, filename)) os.remove(join(dirn, f"links{fileIndex}"))
class Cache(): def __init__(self, cachefilename, CACHE_SIZE, logger=logger('detoursCache.log')): self.lock = threading.RLock() self.cachefilename = cachefilename self.entry = LRUCache(maxsize=CACHE_SIZE) self.logger = logger self.hitcount = 0 def hit(self): self.lock.acquire(blocking=1) try: self.hitcount += 1 finally: self.lock.release() def reset(self): self.lock.acquire(blocking=1) try: self.hitcount = 0 finally: self.lock.release() def push(self, key, val): self.lock.acquire(blocking=1) try: self.entry[key] = val except: return finally: self.lock.release() def get(self, key): self.lock.acquire(blocking=1) try: return self.entry[key] except: return False finally: self.lock.release() def write_to_disk(self): self.lock.acquire(blocking=1) try: cachefile = open(self.cachefilename, 'w') for key, val in self.entry.items(): print(key + '\t' + val, file=cachefile) cachefile.close() finally: self.lock.release() def load_from_disk(self): self.lock.acquire(blocking=1) try: if os.path.exists(self.cachefilename): with open(self.cachefilename, 'r') as f: for line in f: if line == "": continue rline = line.strip() splitvals = rline.split('\t') if len(splitvals) == 2: key = splitvals[0] valstr = splitvals[1] self.entry[key] = valstr else: continue except: self.logger.error("Failed to read existing cache file") raise ("Error in loading previous cache file") finally: self.lock.release()
class CoapLRUCache(CoapCache): def __init__(self, max_dim): """ :param max_dim: """ self.cache = LRUCache(maxsize=max_dim) def update(self, key, element): """ :param key: :param element: :return: """ print "updating cache" print "key: ", key.hashkey print "element: ", element self.cache.update([(key.hashkey, element)]) def get(self, key): """ :param key: :return: CacheElement """ try: response = self.cache[key.hashkey] except KeyError: print "problem here" response = None return response def is_full(self): """ :return: """ if self.cache.currsize == self.cache.maxsize: return True return False def is_empty(self): """ :return: """ if self.cache.currsize == 0: return True return False def __str__(self): msg = [] for e in self.cache.values(): msg.append(str(e)) return "Cache Size: {sz}\n" + "\n".join(msg) def debug_print(self): """ :return: """ print "size = ", self.cache.currsize list = self.cache.items() for key, element in list: print "element.max age ", element.max_age print "element.uri", element.uri print "element.freshness ", element.freshness