class CoapLRUCache(Cache):
def __init__(self, mode, max_dim):
"""
Initialise an LRU cache for the Coap.
:param max_dim: max number of elements in the cache
:param mode: used to differentiate between a cache used in a forward-proxy or in a reverse-proxy
"""
Cache.__init__(self, mode, max_dim)
self.cache = LRUCache(maxsize=max_dim)
def __str__(self):
msg = []
for e in list(self.cache.values()):
msg.append(str(e))
return ("Cache Size: {sz}\n" + "\n".join(msg))
def debug_print(self):
"""
:return: a debug printout for the current cache.
"""
return ("size = %s\n%s" % (
self.cache.currsize,
'\n'.join([
( "element.max age %s\n"\
"element.uri %s\n"\
"element.freshness %s" ) % (
element.max_age,
element.uri,
element.freshness )
for key, element
in list(self.cache.items())
])))
def fixLinks(self, linkTargetSet):
import gc
from cachetools import LRUCache
gc.collect()
dirn = self._filename
filenameList = self._filenameList
fileByWord = {}
for line in open(join(dirn, "index.txt"), encoding="utf-8"):
line = line.rstrip("\n")
if not line:
continue
entryIndex, wordEsc, filename, _ = line.split("\t")
entryIndex = int(entryIndex)
# entryId = f"entry{entryIndex}"
word = unescapeNTB(wordEsc)
if word not in linkTargetSet:
continue
if word in fileByWord:
# log.info(f'fileByWord[{word}]={fileByWord[word]}, filename={filename}')
fileByWord[word].append((filename, entryIndex))
else:
fileByWord[word] = [(filename, entryIndex)]
linksByFile = LRUCache(maxsize=100)
# with open(join(dirn, "fileByWord.json"), "w") as fileByWordFile:
# json.dump(fileByWord, fileByWordFile, ensure_ascii=False, indent="\t")
def getLinksByFile(fileIndex):
_file = linksByFile.get(fileIndex)
if _file is not None:
return _file
_file = open(
join(dirn, f"links{fileIndex}"),
mode="a",
encoding="utf-8",
)
linksByFile[fileIndex] = _file
return _file
log.info("")
for line in open(join(dirn, "links.txt"), encoding="utf-8"):
line = line.rstrip("\n")
if not line:
continue
target, fileIndex, x_start, x_size = line.split("\t")
target = unescapeNTB(target)
if target not in fileByWord:
targetNew = ""
else:
targetFilename, targetEntryIndex = fileByWord[target][0]
if targetFilename == filename:
continue
targetNew = f"{targetFilename}#entry{targetEntryIndex}"
_file = getLinksByFile(int(fileIndex))
_file.write(f"{x_start}\t{x_size}\t{targetNew}\n")
_file.flush()
for _, _file in linksByFile.items():
_file.close()
del linksByFile
linkTargetSet.clear()
del fileByWord, linkTargetSet
gc.collect()
entry_url_fmt = self._glos.getInfo("entry_url")
re_href = re.compile(
b' href="[^<>"]*?"',
re.I,
)
for fileIndex, filename in enumerate(filenameList):
if not isfile(join(dirn, f"links{fileIndex}")):
continue
with open(join(dirn, filename), mode="rb") as inFile:
with open(join(dirn, f"{filename}.new"), mode="wb") as outFile:
for linkLine in open(join(dirn, f"links{fileIndex}"),
"rb"):
outFile.flush()
linkLine = linkLine.rstrip(b"\n")
x_start, x_size, target = linkLine.split(b"\t")
outFile.write(
inFile.read(int(x_start, 16) - inFile.tell()))
curLink = inFile.read(int(x_size, 16))
if target:
outFile.write(
re_href.sub(
b' href="./' + target + b'"',
curLink,
))
continue
if not entry_url_fmt:
outFile.write(
curLink.replace(
b' href="#',
b' class="broken" href="#',
))
continue
_st = curLink.decode("utf-8")
i = _st.find('href="#')
j = _st.find('"', i + 7)
word = _st[i + 7:j]
url = entry_url_fmt.format(word=word)
outFile.write(
(_st[:i] + f'class="broken" href="{url}"' +
_st[j + 1:]).encode("utf-8"))
outFile.write(inFile.read())
os.rename(join(dirn, f"{filename}.new"), join(dirn, filename))
os.remove(join(dirn, f"links{fileIndex}"))
class Server(Component):
def init(self, args, db, hosts, logger):
self.args = args
self.db = db
self.hosts = hosts
self.logger = logger
self.bind = args.bind
self.forward = args.forward
self.peers = {}
self.requests = {}
self.cache = LRUCache(maxsize=args.cachesize)
if args.daemon:
Daemon(args.pidfile).register(self)
if args.debug:
Debugger(events=args.verbose, logger=logger).register(self)
self.transport = UDPServer(self.bind).register(self)
self.protocol = DNS().register(self)
def ready(self, server, bind):
self.logger.info(
"DNS Server Ready! Listening on {0:s}:{1:d}".format(*bind))
Timer(1, Event.create("ttl"), persist=True).register(self)
def ttl(self):
for k, rrs in self.cache.items()[:]:
if any(rr.ttl == 0 for rr in rrs):
qname, qtype, qclass = k
self.logger.info("Expired Entry: {0:s} {1:s} {2:s}".format(
CLASS.get(qclass), QTYPE.get(qtype), qname))
del self.cache[k]
else:
for rr in rrs:
rr.ttl -= 1
def request(self, peer, request):
qname = str(request.q.qname)
qtype = request.q.qtype
qclass = request.q.qclass
key = (qname, qtype, qclass)
if key in self.cache:
self.logger.info(
"Cached Request ({0:s}): {1:s} {2:s} {3:s}".format(
"{0:s}:{1:d}".format(*peer), CLASS.get(qclass),
QTYPE.get(qtype), qname))
reply = request.reply()
for rr in self.cache[key]:
reply.add_answer(rr)
self.fire(write(peer, reply.pack()))
return
if key in self.hosts:
self.logger.info(
"Local Hosts Request ({0:s}): {1:s} {2:s} {3:s}".format(
"{0:s}:{1:d}".format(*peer), CLASS.get(qclass),
QTYPE.get(qtype), qname))
rr = [RR(qname, rdata=A(self.hosts[key]))]
reply = request.reply()
reply.add_answer(*rr)
self.cache[key] = rr
self.fire(write(peer, reply.pack()))
return
records = Record.objects.filter(rname=qname,
rclass=qclass,
rtype=qtype)
if records:
self.logger.info(
"Authoritative Request ({0:s}): {1:s} {2:s} {3:s}".format(
"{0:s}:{1:d}".format(*peer), CLASS.get(qclass),
QTYPE.get(qtype), qname))
rr = [record.rr for record in records]
reply = request.reply()
reply.add_answer(*rr)
self.cache[key] = rr
self.fire(write(peer, reply.pack()))
return
self.logger.info("Request ({0:s}): {1:s} {2:s} {3:s}".format(
"{0:s}:{1:d}".format(*peer), CLASS.get(qclass), QTYPE.get(qtype),
qname))
lookup = DNSRecord(q=DNSQuestion(qname, qtype, qclass))
id = lookup.header.id
self.peers[id] = peer
self.requests[id] = request
self.fire(write((self.forward, 53), lookup.pack()))
def response(self, peer, response):
id = response.header.id
qname = str(response.q.qname)
qtype = response.q.qtype
qclass = response.q.qclass
if id not in self.peers:
self.logger.info(
"Unknown Response ({0:s}): {1:s} {2:s} {3:s}".format(
"{0:s}:{1:d}".format(*peer), CLASS.get(qclass),
QTYPE.get(qtype), qname))
return
peer = self.peers[id]
request = self.requests[id]
key = (str(request.q.qname), request.q.qtype, request.q.qclass)
reply = request.reply()
reply.add_answer(*response.rr)
self.cache[key] = reply.rr
self.fire(write(peer, reply.pack()))
del self.peers[id]
del self.requests[id]
class CoapLRUCache(CoapCache):
def __init__(self, max_dim):
"""
:param max_dim:
"""
self.cache = LRUCache(maxsize=max_dim)
def update(self, key, element):
"""
:param key:
:param element:
:return:
"""
logger.debug("updating cache, key: %s, element: %s", \
key.hashkey, element)
self.cache.update([(key.hashkey, element)])
def get(self, key):
"""
:param key:
:return: CacheElement
"""
try:
response = self.cache[key.hashkey]
except KeyError:
logger.debug("problem here", exc_info=1)
response = None
return response
def is_full(self):
"""
:return:
"""
if self.cache.currsize == self.cache.maxsize:
return True
return False
def is_empty(self):
"""
:return:
"""
if self.cache.currsize == 0:
return True
return False
def __str__(self):
msg = []
for e in self.cache.values():
msg.append(str(e))
return "Cache Size: {sz}\n" + "\n".join(msg)
def debug_print(self):
"""
:return:
"""
return ("size = %s\n%s" % (
self.cache.currsize,
'\n'.join([
( "element.max age %s\n"\
"element.uri %s\n"\
"element.freshness %s" ) % (
element.max_age,
element.uri,
element.freshness )
for key, element
in list(self.cache.items())
])))
class Server(Component):
channel = "server"
def init(self, args, db, hosts, logger):
self.args = args
self.db = db
self.hosts = hosts
self.logger = logger
self.bind = args.bind
self.forward = args.forward
self.peers = {}
self.requests = {}
self.cache = LRUCache(maxsize=args.cachesize)
if args.daemon:
Daemon(args.pidfile).register(self)
if args.debug:
Debugger(events=args.verbose, logger=logger).register(self)
self.transport = UDPServer(
self.bind, channel=self.channel
).register(self)
self.protocol = DNS(channel=self.channel).register(self)
def ready(self, server, bind):
self.logger.info(
"DNS Server Ready! Listening on {0:s}:{1:d}".format(*bind)
)
# Timer(1, Event.create("ttl"), persist=True, channel=self.channel).register(self)
def ttl(self):
for k, rrs in self.cache.items()[:]:
if any(rr.ttl == 0 for rr in rrs):
qname, qtype, qclass = k
self.logger.info(
"Expired Entry: {0:s} {1:s} {2:s}".format(
CLASS.get(qclass), QTYPE.get(qtype), qname
)
)
del self.cache[k]
else:
for rr in rrs:
rr.ttl -= 1
def request(self, peer, request):
qname = str(request.q.qname)
qtype = request.q.qtype
qclass = request.q.qclass
key = (qname, qtype, qclass)
if key in self.cache:
self.logger.info(
"Cached Request ({0:s}): {1:s} {2:s} {3:s}".format(
"{0:s}:{1:d}".format(*peer),
CLASS.get(qclass), QTYPE.get(qtype), qname
)
)
reply = request.reply()
for rr in self.cache[key]:
reply.add_answer(rr)
self.fire(write(peer, reply.pack()))
return
if key in self.hosts:
self.logger.info(
"Local Hosts Request ({0:s}): {1:s} {2:s} {3:s}".format(
"{0:s}:{1:d}".format(*peer),
CLASS.get(qclass), QTYPE.get(qtype), qname
)
)
reply = request.reply()
for rdata in self.hosts[key]:
rr = RR(
qname,
rclass=CLASS.IN,
rtype=QTYPE.AAAA if ":" in rdata else QTYPE.A,
rdata=AAAA(rdata) if ":" in rdata else A(rdata)
)
reply.add_answer(rr)
self.cache[key] = rr
self.fire(write(peer, reply.pack()))
return
records = Record.objects.filter(rname=qname)
if not records:
self.logger.info(
"Request ({0:s}): {1:s} {2:s} {3:s} -> {4:s}:{5:d}".format(
"{0:s}:{1:d}".format(*peer),
CLASS.get(qclass), QTYPE.get(qtype), qname,
self.forward, 53
)
)
lookup = DNSRecord(q=DNSQuestion(qname, qtype, qclass))
id = lookup.header.id
self.peers[id] = peer
self.requests[id] = request
self.fire(write((self.forward, 53), lookup.pack()))
return
self.logger.info(
"Authoritative Request ({0:s}): {1:s} {2:s} {3:s}".format(
"{0:s}:{1:d}".format(*peer),
CLASS.get(qclass), QTYPE.get(qtype), qname
)
)
rr = []
reply = request.reply()
if len(records) == 1 and records[0].rtype == CNAME:
rr.append(records[0].rr)
records = Record.objects.filter(rname=records[0].rdata)
for record in records:
rr.append(record.rr)
reply.add_answer(*rr)
self.cache[key] = rr
self.fire(write(peer, reply.pack()))
def response(self, peer, response):
id = response.header.id
qname = str(response.q.qname)
qtype = response.q.qtype
qclass = response.q.qclass
if id not in self.peers:
self.logger.info(
"Unknown Response ({0:s}): {1:s} {2:s} {3:s}".format(
"{0:s}:{1:d}".format(*peer),
CLASS.get(qclass), QTYPE.get(qtype), qname
)
)
return
peer = self.peers[id]
request = self.requests[id]
key = (str(request.q.qname), request.q.qtype, request.q.qclass)
reply = request.reply()
reply.add_answer(*response.rr)
self.cache[key] = reply.rr
self.fire(write(peer, reply.pack()))
del self.peers[id]
del self.requests[id]
class CoapLRUCache(CoapCache):
def __init__(self, max_dim):
"""
:param max_dim:
"""
self.cache = LRUCache(maxsize=max_dim)
def update(self, key, element):
"""
:param key:
:param element:
:return:
"""
six.print_("updating cache")
six.print_("key: ", key.hashkey)
six.print_("element: ", element)
self.cache.update([(key.hashkey, element)])
def get(self, key):
"""
:param key:
:return: CacheElement
"""
try:
response = self.cache[key.hashkey]
except KeyError:
six.print_("problem here")
response = None
return response
def is_full(self):
"""
:return:
"""
if self.cache.currsize == self.cache.maxsize:
return True
return False
def is_empty(self):
"""
:return:
"""
if self.cache.currsize == 0:
return True
return False
def debug_print(self):
"""
:return:
"""
six.print_("size = ", self.cache.currsize)
list = self.cache.items()
for key, element in list:
six.print_("element.max age ", element.max_age)
six.print_("element.uri", element.uri)
six.print_("element.freshness ", element.freshness)
class Cache():
def __init__(self,cachefilename,CACHE_SIZE,logger=logger('detoursCache.log')):
self.lock = threading.RLock()
self.cachefilename = cachefilename
self.entry = LRUCache(maxsize=CACHE_SIZE)
self.logger=logger
self.hitcount=0
def hit(self):
self.lock.acquire(blocking=1)
try:
self.hitcount+=1
finally:
self.lock.release()
def reset(self):
self.lock.acquire(blocking=1)
try:
self.hitcount=0
finally:
self.lock.release()
def push(self,key,val):
self.lock.acquire(blocking=1)
try:
self.entry[key]=val
except:
return
finally:
self.lock.release()
def get(self,key):
self.lock.acquire(blocking=1)
try:
return self.entry[key]
except:
return False
finally:
self.lock.release()
def write_to_disk(self):
self.lock.acquire(blocking=1)
try:
cachefile = open(self.cachefilename,'w')
for key,val in self.entry.items():
print(key+'\t'+val,file=cachefile)
cachefile.close()
finally:
self.lock.release()
def load_from_disk(self):
self.lock.acquire(blocking=1)
try:
if os.path.exists(self.cachefilename):
with open(self.cachefilename, 'r') as f:
for line in f:
if line == "":
continue
rline = line.strip()
splitvals=rline.split('\t')
if len(splitvals) == 2:
key=splitvals[0]
valstr=splitvals[1]
self.entry[key]=valstr
else:
continue
except:
self.logger.error("Failed to read existing cache file")
raise("Error in loading previous cache file")
finally:
self.lock.release()
def fixLinks(self, linkTargetSet):
import gc
from cachetools import LRUCache
gc.collect()
dirn = self._filename
filenameList = self._filenameList
fileByWord = {}
for line in open(join(dirn, "index.txt"), encoding="utf-8"):
line = line.rstrip("\n")
if not line:
continue
word, filename, _ = line.split("\t")
word = unescapeNTB(word)
if word not in linkTargetSet:
continue
fileByWord[word] = filename
linksByFile = LRUCache(maxsize=100)
def getLinksByFile(fileIndex):
_file = linksByFile.get(fileIndex)
if _file is not None:
return _file
_file = open(
join(dirn, f"links{fileIndex}"),
mode="a",
encoding="utf-8",
)
linksByFile[fileIndex] = _file
return _file
log.info("")
for line in open(join(dirn, "links.txt"), encoding="utf-8"):
line = line.rstrip("\n")
if not line:
continue
target, fileIndex, x_start, x_size = line.split("\t")
target = unescapeNTB(target)
if target not in fileByWord:
targetFilename = ""
else:
targetFilename = fileByWord[target]
if targetFilename == filename:
continue
_file = getLinksByFile(int(fileIndex))
_file.write(
f"{x_start}\t{x_size}\t{targetFilename}\n"
)
_file.flush()
for _, _file in linksByFile.items():
_file.close()
del linksByFile
linkTargetSet.clear()
del fileByWord, linkTargetSet
gc.collect()
entry_url_fmt = self._glos.getInfo("entry_url")
for fileIndex, filename in enumerate(filenameList):
with open(join(dirn, filename), mode="rb") as inFile:
with open(join(dirn, f"{filename}.new"), mode="wb") as outFile:
for linkLine in open(join(dirn, f"links{fileIndex}"), "rb"):
outFile.flush()
linkLine = linkLine.rstrip(b"\n")
x_start, x_size, targetFilename = linkLine.split(b"\t")
outFile.write(inFile.read(
int(x_start, 16) - inFile.tell()
))
curLink = inFile.read(int(x_size, 16))
if targetFilename:
outFile.write(curLink.replace(
b' href="#',
b' href="./' + targetFilename + b'#',
))
continue
if not entry_url_fmt:
outFile.write(curLink.replace(
b' href="#',
b' class="broken" href="#',
))
continue
_st = curLink.decode("utf-8")
i = _st.find('href="#')
j = _st.find('"', i + 7)
word = _st[i + 7:j]
url = entry_url_fmt.format(word=word)
outFile.write((
_st[:i] +
f'class="broken" href="{url}"' +
_st[j + 1:]
).encode("utf-8"))
outFile.write(inFile.read())
os.rename(join(dirn, f"{filename}.new"), join(dirn, filename))
os.remove(join(dirn, f"links{fileIndex}"))
class Cache():
def __init__(self,
cachefilename,
CACHE_SIZE,
logger=logger('detoursCache.log')):
self.lock = threading.RLock()
self.cachefilename = cachefilename
self.entry = LRUCache(maxsize=CACHE_SIZE)
self.logger = logger
self.hitcount = 0
def hit(self):
self.lock.acquire(blocking=1)
try:
self.hitcount += 1
finally:
self.lock.release()
def reset(self):
self.lock.acquire(blocking=1)
try:
self.hitcount = 0
finally:
self.lock.release()
def push(self, key, val):
self.lock.acquire(blocking=1)
try:
self.entry[key] = val
except:
return
finally:
self.lock.release()
def get(self, key):
self.lock.acquire(blocking=1)
try:
return self.entry[key]
except:
return False
finally:
self.lock.release()
def write_to_disk(self):
self.lock.acquire(blocking=1)
try:
cachefile = open(self.cachefilename, 'w')
for key, val in self.entry.items():
print(key + '\t' + val, file=cachefile)
cachefile.close()
finally:
self.lock.release()
def load_from_disk(self):
self.lock.acquire(blocking=1)
try:
if os.path.exists(self.cachefilename):
with open(self.cachefilename, 'r') as f:
for line in f:
if line == "":
continue
rline = line.strip()
splitvals = rline.split('\t')
if len(splitvals) == 2:
key = splitvals[0]
valstr = splitvals[1]
self.entry[key] = valstr
else:
continue
except:
self.logger.error("Failed to read existing cache file")
raise ("Error in loading previous cache file")
finally:
self.lock.release()
class CoapLRUCache(CoapCache):
def __init__(self, max_dim):
"""
:param max_dim:
"""
self.cache = LRUCache(maxsize=max_dim)
def update(self, key, element):
"""
:param key:
:param element:
:return:
"""
print "updating cache"
print "key: ", key.hashkey
print "element: ", element
self.cache.update([(key.hashkey, element)])
def get(self, key):
"""
:param key:
:return: CacheElement
"""
try:
response = self.cache[key.hashkey]
except KeyError:
print "problem here"
response = None
return response
def is_full(self):
"""
:return:
"""
if self.cache.currsize == self.cache.maxsize:
return True
return False
def is_empty(self):
"""
:return:
"""
if self.cache.currsize == 0:
return True
return False
def __str__(self):
msg = []
for e in self.cache.values():
msg.append(str(e))
return "Cache Size: {sz}\n" + "\n".join(msg)
def debug_print(self):
"""
:return:
"""
print "size = ", self.cache.currsize
list = self.cache.items()
for key, element in list:
print "element.max age ", element.max_age
print "element.uri", element.uri
print "element.freshness ", element.freshness
