def __init__(self, config, fastBundle, slowBundle, exitSignal):
with DebugInfo("Initialize buffer bundle"):
self.config = config
from histo.bundle import Safe
self.exitSignal = exitSignal
self.fastBundle = Safe(fastBundle)
from histo.bundle import ListCache
self.slowBundle = ListCache(slowBundle, config["ListInterval"], exitSignal)
from threading import RLock
self.lock = RLock()
self.usageLog = self.getUsageLog(config["UsageLogFile"])
self.queue = TaskQueue()
self.threads = self.createTransferThreads(config["ThreadCount"])
self.protectedFiles = []
for e in self.threads:
e.start()
def supplyNotUploaded():
self.queue.extend(self.getNotUploadedFiles())
from threading import Thread
Thread(target=supplyNotUploaded).start()
class Buffer:
def __init__(self, config, fastBundle, slowBundle, exitSignal):
with DebugInfo("Initialize buffer bundle"):
self.config = config
from histo.bundle import Safe
self.exitSignal = exitSignal
self.fastBundle = Safe(fastBundle)
from histo.bundle import ListCache
self.slowBundle = ListCache(slowBundle, config["ListInterval"], exitSignal)
from threading import RLock
self.lock = RLock()
self.usageLog = self.getUsageLog(config["UsageLogFile"])
self.queue = TaskQueue()
self.threads = self.createTransferThreads(config["ThreadCount"])
self.protectedFiles = []
for e in self.threads:
e.start()
def supplyNotUploaded():
self.queue.extend(self.getNotUploadedFiles())
from threading import Thread
Thread(target=supplyNotUploaded).start()
def open(self, name, mode):
with self.lock:
with DebugInfo("Open %s with mode %r" % (name, mode)):
if mode == "wb":
return self.openForWrite(name)
elif mode == "rb":
return self.openForRead(name)
else:
raise Exception("No such mode.")
def list(self):
with DebugInfo("List") as d:
with self.lock:
result = set()
result.union()
slowFiles = self.slowBundle.list()
fastFiles = self.fastBundle.list()
logger.debug("Slow files: %d" % len(slowFiles))
logger.debug("Fast files: %d" % len(fastFiles))
result = result.union(slowFiles)
result = result.union(fastFiles)
result = list(result)
d.result = str(len(result))
return result
def listFast(self):
with DebugInfo("List fast"):
return self.fastBundle.list()
def delete(self, name):
raise Exception("Not support.")
def protect(self, name):
with DebugInfo("Protect %s" % name):
self.protectedFiles.append(name)
def unprotect(self, name):
with DebugInfo("Unprotect: %s" % name):
self.protectedFiles.remove(name)
def getNotUploadedFiles(self):
with DebugInfo("Get not uploaded files") as d:
fastFiles = self.fastBundle.list()
slowFiles = self.slowBundle.list()
for e in slowFiles:
if e in fastFiles:
fastFiles.remove(e)
d.result = "Count: %d" % len(fastFiles)
return fastFiles
def getUsageLog(self, usageLogFile):
with DebugInfo("Get usage log"):
return UsageLog(usageLogFile)
def createTransferThreads(self, threadCount):
with DebugInfo("Create transfer threads"):
return [self.createTransferThread() for _ in range(threadCount)]
def openForWrite(self, name):
with DebugInfo("Open for write %s" % name):
result = self.fastBundle.open(name, "wb")
def onClose(close0):
with DebugInfo("Close %s" % name):
with self.lock:
self.queue.append(name)
close0()
self.limitBufferSize()
return FileHook(result, onClose=onClose)
def openForRead(self, name):
with DebugInfo("Open for read: %s" % name) as d:
self.usageLog.log(name)
if self.fastBundle.exists(name):
d.result = "From cache"
return self.fastBundle.open(name, "rb")
else:
d.result = "From slow"
return self.openSlowBundleForRead(name)
def openSlowBundleForRead(self, name):
result = FileShell()
def threadProc():
with DebugInfo("Read slow file %s" % name) as d:
from histo.bundle.safe import SafeProtection
try:
file = self.fetchSlowFileForRead(name)
result.fill(file)
except SafeProtection as e:
result.fail()
d.result = "Fail %s" % repr(e)
except Exception as e:
result.fail()
d.result = "Fail %s" % repr(e)
with DebugInfo("Start read slow file thread"):
from threading import Thread
Thread(target=threadProc).start()
return result
def fetchSlowFileForRead(self, name):
with DebugInfo("Fetching slow file %s" % name):
with DebugInfo("Open slow file"):
slowFile = self.slowBundle.open(name, "rb")
with slowFile as f1:
with DebugInfo("Protect %s" % name):
protection = self.fastBundle.protect(name)
with protection:
with DebugInfo("Open fast file"):
fastFile = self.fastBundle.openIgnoreProtection(name, "wb")
with fastFile as f2:
with DebugInfo("Copy"):
from pclib import copystream2
copystream2(f1, f2)
with DebugInfo("Reopen fast file"):
result = self.fastBundle.openIgnoreProtection(name, "rb")
return result
def createTransferThread(self):
with DebugInfo("Create transfer thread"):
return TransferThread(self.fastBundle, self.slowBundle, self.queue, self.exitSignal)
def limitBufferSize(self):
with DebugInfo("Limit buffer size"):
with self.lock:
currentBufferSize = self.fastBundle.getTotalSize()
mostUseless = self.getMostUseless()
for e in mostUseless:
if currentBufferSize <= self.config["MaxBufferSize"]:
break
fileSize = self.fastBundle.getSize(e)
if self.deleteCache(e):
currentBufferSize -= fileSize
def deleteCache(self, file):
with DebugInfo("Delete cache"):
if file in self.protectedFiles:
logger.debug("%s is in protection, should not be deleted" % file)
return False
if file not in self.slowBundle.listWithCache():
logger.debug("%s may not uploaded, it should not be deleted" % file)
return False
from histo.bundle.safe import SafeProtection
try:
self.fastBundle.delete(file)
logger.debug("Delete %s ok" % file)
return True
except SafeProtection as e:
logger.debug("Delete %s failed: %s" % (file, repr(e)))
return False
except Exception as e:
logger.exception(e)
logger.debug("Delete %s failed" % file)
return False
def getMostUseless(self):
with DebugInfo("Get most useless"):
files = self.fastBundle.list()
files = [(-self.usageLog.getUsageCount(e), e) for e in files]
files = sorted(files)
files = [e[1] for e in files]
return files