Example #1
0
    def __init__(self, config, fastBundle, slowBundle, exitSignal):
        with DebugInfo("Initialize buffer bundle"):
            self.config = config
            from histo.bundle import Safe

            self.exitSignal = exitSignal
            self.fastBundle = Safe(fastBundle)
            from histo.bundle import ListCache

            self.slowBundle = ListCache(slowBundle, config["ListInterval"], exitSignal)
            from threading import RLock

            self.lock = RLock()
            self.usageLog = self.getUsageLog(config["UsageLogFile"])
            self.queue = TaskQueue()
            self.threads = self.createTransferThreads(config["ThreadCount"])
            self.protectedFiles = []
            for e in self.threads:
                e.start()

            def supplyNotUploaded():
                self.queue.extend(self.getNotUploadedFiles())

            from threading import Thread

            Thread(target=supplyNotUploaded).start()
Example #2
0
class Buffer:
    def __init__(self, config, fastBundle, slowBundle, exitSignal):
        with DebugInfo("Initialize buffer bundle"):
            self.config = config
            from histo.bundle import Safe

            self.exitSignal = exitSignal
            self.fastBundle = Safe(fastBundle)
            from histo.bundle import ListCache

            self.slowBundle = ListCache(slowBundle, config["ListInterval"], exitSignal)
            from threading import RLock

            self.lock = RLock()
            self.usageLog = self.getUsageLog(config["UsageLogFile"])
            self.queue = TaskQueue()
            self.threads = self.createTransferThreads(config["ThreadCount"])
            self.protectedFiles = []
            for e in self.threads:
                e.start()

            def supplyNotUploaded():
                self.queue.extend(self.getNotUploadedFiles())

            from threading import Thread

            Thread(target=supplyNotUploaded).start()

    def open(self, name, mode):
        with self.lock:
            with DebugInfo("Open %s with mode %r" % (name, mode)):
                if mode == "wb":
                    return self.openForWrite(name)
                elif mode == "rb":
                    return self.openForRead(name)
                else:
                    raise Exception("No such mode.")

    def list(self):
        with DebugInfo("List") as d:
            with self.lock:
                result = set()
                result.union()
                slowFiles = self.slowBundle.list()
                fastFiles = self.fastBundle.list()
                logger.debug("Slow files: %d" % len(slowFiles))
                logger.debug("Fast files: %d" % len(fastFiles))
                result = result.union(slowFiles)
                result = result.union(fastFiles)
                result = list(result)
                d.result = str(len(result))
                return result

    def listFast(self):
        with DebugInfo("List fast"):
            return self.fastBundle.list()

    def delete(self, name):
        raise Exception("Not support.")

    def protect(self, name):
        with DebugInfo("Protect %s" % name):
            self.protectedFiles.append(name)

    def unprotect(self, name):
        with DebugInfo("Unprotect: %s" % name):
            self.protectedFiles.remove(name)

    def getNotUploadedFiles(self):
        with DebugInfo("Get not uploaded files") as d:
            fastFiles = self.fastBundle.list()
            slowFiles = self.slowBundle.list()
            for e in slowFiles:
                if e in fastFiles:
                    fastFiles.remove(e)
            d.result = "Count: %d" % len(fastFiles)
            return fastFiles

    def getUsageLog(self, usageLogFile):
        with DebugInfo("Get usage log"):
            return UsageLog(usageLogFile)

    def createTransferThreads(self, threadCount):
        with DebugInfo("Create transfer threads"):
            return [self.createTransferThread() for _ in range(threadCount)]

    def openForWrite(self, name):
        with DebugInfo("Open for write %s" % name):
            result = self.fastBundle.open(name, "wb")

            def onClose(close0):
                with DebugInfo("Close %s" % name):
                    with self.lock:
                        self.queue.append(name)
                        close0()
                        self.limitBufferSize()

            return FileHook(result, onClose=onClose)

    def openForRead(self, name):
        with DebugInfo("Open for read: %s" % name) as d:
            self.usageLog.log(name)
            if self.fastBundle.exists(name):
                d.result = "From cache"
                return self.fastBundle.open(name, "rb")
            else:
                d.result = "From slow"
                return self.openSlowBundleForRead(name)

    def openSlowBundleForRead(self, name):
        result = FileShell()

        def threadProc():
            with DebugInfo("Read slow file %s" % name) as d:
                from histo.bundle.safe import SafeProtection

                try:
                    file = self.fetchSlowFileForRead(name)
                    result.fill(file)
                except SafeProtection as e:
                    result.fail()
                    d.result = "Fail %s" % repr(e)
                except Exception as e:
                    result.fail()
                    d.result = "Fail %s" % repr(e)

        with DebugInfo("Start read slow file thread"):
            from threading import Thread

            Thread(target=threadProc).start()
            return result

    def fetchSlowFileForRead(self, name):
        with DebugInfo("Fetching slow file %s" % name):
            with DebugInfo("Open slow file"):
                slowFile = self.slowBundle.open(name, "rb")
            with slowFile as f1:
                with DebugInfo("Protect %s" % name):
                    protection = self.fastBundle.protect(name)
                with protection:
                    with DebugInfo("Open fast file"):
                        fastFile = self.fastBundle.openIgnoreProtection(name, "wb")
                    with fastFile as f2:
                        with DebugInfo("Copy"):
                            from pclib import copystream2

                            copystream2(f1, f2)
                    with DebugInfo("Reopen fast file"):
                        result = self.fastBundle.openIgnoreProtection(name, "rb")
                    return result

    def createTransferThread(self):
        with DebugInfo("Create transfer thread"):
            return TransferThread(self.fastBundle, self.slowBundle, self.queue, self.exitSignal)

    def limitBufferSize(self):
        with DebugInfo("Limit buffer size"):
            with self.lock:
                currentBufferSize = self.fastBundle.getTotalSize()
                mostUseless = self.getMostUseless()
                for e in mostUseless:
                    if currentBufferSize <= self.config["MaxBufferSize"]:
                        break
                    fileSize = self.fastBundle.getSize(e)
                    if self.deleteCache(e):
                        currentBufferSize -= fileSize

    def deleteCache(self, file):
        with DebugInfo("Delete cache"):
            if file in self.protectedFiles:
                logger.debug("%s is in protection, should not be deleted" % file)
                return False
            if file not in self.slowBundle.listWithCache():
                logger.debug("%s may not uploaded, it should not be deleted" % file)
                return False
            from histo.bundle.safe import SafeProtection

            try:
                self.fastBundle.delete(file)
                logger.debug("Delete %s ok" % file)
                return True
            except SafeProtection as e:
                logger.debug("Delete %s failed: %s" % (file, repr(e)))
                return False
            except Exception as e:
                logger.exception(e)
                logger.debug("Delete %s failed" % file)
                return False

    def getMostUseless(self):
        with DebugInfo("Get most useless"):
            files = self.fastBundle.list()
            files = [(-self.usageLog.getUsageCount(e), e) for e in files]
            files = sorted(files)
            files = [e[1] for e in files]
            return files