def __init__(self, args): props = args['properties'] self._sourceDir = props.get('path') cacheDir = props.get('cache-dir') cacheSizeInMB = props.get('cache-size') if cacheSizeInMB is not None: cacheSize = cacheSizeInMB * 10**6 # in bytes else: cacheSize = None cleanupEnabled = props.get('cleanup-enabled') cleanupHighWatermark = props.get('cleanup-high-watermark') cleanupLowWatermark = props.get('cleanup-low-watermark') self._sessions = {} # {CopySession: None} self._index = {} # {path: CopySession} self.stats = cachestats.CacheStatistics() self.cache = cachemanager.CacheManager(self.stats, cacheDir, cacheSize, cleanupEnabled, cleanupHighWatermark, cleanupLowWatermark) common.ensureDir(self._sourceDir, "source") # Startup copy thread self._thread = CopyThread(self)
def testConflictMTime(self): def writeContent(f, content): f.write(content) return f m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True, 0.4, 0.2) # create a cache for "file' with some old content d = m.newTempFile("file", 1024, 1256040206) d.addCallback(writeContent, "old\n") d.addCallback(lambda f: setattr(self, "_file", f)) # create a second cache for "file' with newer content d.addCallback(lambda _: m.newTempFile("file", 1024, 1256040207)) d.addCallback(writeContent, "new\n") # and complete it now d.addCallback(self.completeAndClose, m) # then, complete first cache (conflict) d.addCallback(lambda _: self._file) d.addCallback(self.completeAndClose, m) d.addCallback(lambda f: setattr(self, "_file", None)) # check that we get back our "new" content after conflict d.addCallback(lambda _: m.openCacheFile("file")) d.addCallback(lambda f: setattr(self, "_file", f)) d.addCallback(lambda _: self._file.readline()) d.addCallback(lambda s: self.assertEqual(s, "new\n")) d.addCallback(lambda _: self._file.close()) return d
def testHitMiss(self): m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True, 0.5, 0.2) # updateCacheUsage ~= 0 d = m.updateCacheUsage() d.addCallback(self.checkUsage, m, 0) d.addCallback(lambda _: self.checkCacheMiss(m, "test3")) d.addCallback(lambda _: self.checkCacheMiss(m, "test4")) d.addCallback( lambda _: m.newTempFile("test3", CACHE_SIZE - MAX_PAGE_SIZE)) d.addCallback(lambda f: setattr(self, "_file", f)) d.addCallback(lambda _: self.checkCacheMiss(m, "test3")) d.addCallback(lambda _: self._file) d.addCallback(self.completeAndClose, m) d.addErrback(self.fail) d.addCallback(lambda _: self.checkCacheHit(m, "test3")) d.addCallback(lambda _: self.checkCacheMiss(m, "test4")) d.addCallback( lambda _: m.newTempFile("test4", CACHE_SIZE - MAX_PAGE_SIZE)) d.addCallback(self.completeAndClose, m) # The cache cleaning is done in a seperate thread... d.addCallback(lambda _: time.sleep(1)) d.addCallback(lambda _: self.checkCacheMiss(m, "test3")) return d
def testLowLevel(self): def _makeTemp(tag, size, m, name): t = cachemanager.TempFile(m, name, tag, size) return t m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True, 0.5, 0.3) # getIdentifier/Path self.assertEquals(type(m.getIdentifier("path")), type("")) self.assertEquals(m.getIdentifier("path"), m.getIdentifier("path")) self.failIfEquals(m.getIdentifier("path"), m.getIdentifier("path2")) self.assertEquals(type(m.getCachePath("path")), type("")) self.assertEquals(type(m.getTempPath("path")), type("")) # updateCacheUsage ~= 0 d = m.updateCacheUsage() d.addCallback(self.checkUsage, m, 0) # allocate&release d.addCallback(lambda _: m.allocateCacheSpace(100 * 1024)) d.addCallback(self._releaseCacheSpace, m) # updateCacheUsage ~= 0 d.addCallback(lambda _: m.updateCacheUsage()) d.addCallback(self.checkUsage, m, 0) # allocate > CACHE_SIZE d.addCallback(lambda _: m.allocateCacheSpace(CACHE_SIZE * 2)) d.addCallback(lambda c: self.failUnless(c is None)) d.addErrback(self.fail) # updateCacheUsage ~= 0 d.addCallback(lambda _: m.updateCacheUsage()) d.addCallback(self.checkUsage, m, 0) # close incomplete TempFile d.addCallback(lambda _: m.allocateCacheSpace(100 * 1024)) d.addCallback(_makeTemp, 100 * 1024, m, "test") d.addCallback(lambda t: t.close()) # updateCacheUsage ~= 0 d.addCallback(lambda _: m.updateCacheUsage()) d.addCallback(self.checkUsage, m, 0) # close complete TempFile d.addCallback(lambda _: m.allocateCacheSpace(100 * 1024)) d.addCallback(_makeTemp, 100 * 1024, m, "test1") d.addCallback(self.completeAndClose, m) # updateCacheUsage ~= 100k d.addCallback(lambda _: m.updateCacheUsage()) d.addCallback(self.checkUsage, m, 100 * 1024) return d
def testMTime(self): m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True, 0.4, 0.2) d = m.newTempFile("test_mtime", 1024, 1256040206) d.addCallback(self.completeAndClose, m) d.addCallback(lambda _: m.openCacheFile("test_mtime")) d.addCallback(lambda f: f.stat[stat.ST_MTIME]) d.addCallback(lambda t: self.assertEqual(t, 1256040206)) return d
def testCheckCompleted(self): m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True, 0.4, 0.2) d = m.newTempFile("test_completed", 5) d.addCallback(self.checkCompleted) d.addCallback(lambda _: m.newTempFile("test_incomplete", 5)) d.addCallback(self.checkIncomplete) return d
def testFileSizeLimit(self): m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True, 0.4, 0.2) d = m.newTempFile("test_limit", 5) d.addCallback(lambda f: f.write("123456")) d.addCallback(self.fail) d.addErrback(lambda f: f.trap(IOError)) d.addCallback(lambda _: None) return d
def testMultiThread(self): # FIXME: this test can deadlock.... return m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True, 0.4, 0.2) dl = [] d = threads.deferToThread(self.fillTestCache2, m) dl.append(d) threads.deferToThread(self.fillTestCache2, m) dl.append(d) threads.deferToThread(self.fillTestCache2, m) dl.append(d) return defer.DeferredList(dl)
def testCacheCleanUp(self): m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True, 0.4, 0.2) self.failIf(m.stats.oncleanup != 0) d = self.fillTestCache(m, CACHE_SIZE * 4) # The cache cleaning is done in a seperate thread... d.addCallback(lambda _: time.sleep(1)) d.addCallback(lambda _: m.updateCacheUsage()) d.addCallback(lambda u: self.failIf(u > CACHE_SIZE / 2)) d.addCallback(lambda _: self.failIf(m.stats.oncleanup < 5)) return d
def testNoCleanUp(self): m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE * 1.1, False, 1.0, 0.2) d = m.newTempFile("FAT FILE", CACHE_SIZE / 2) d.addCallback(self.completeAndClose, m) d.addCallback(lambda _: m.newTempFile("FAT FILE TWIN", CACHE_SIZE / 2)) d.addCallback(self.completeAndClose, m) d.addCallback(lambda _: m.newTempFile("FAT FILE BRO'", CACHE_SIZE / 2)) d.addCallback(lambda t: self.failUnless(t is None)) d.addCallback(lambda _: self.failIf(m.stats.oncleanup != 0)) return d
def testUnlink(self): m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, False, 0.5, 0.2) name = "unlink-me" path = m.getCachePath(name) d = m.newTempFile(name, 1024, 1256040206) # TODO: d.addCallback(lambda f: f.complete) is not enough to # pass the test, weirdo... d.addCallback(self.completeAndClose, m) # check that unlink actually works d.addCallback(lambda _: m.openCacheFile(name)) d.addErrback(lambda _: self.fail) d.addCallback(lambda c: c.unlink()) d.addCallback(lambda _: os.stat(path)) d.addCallback(self.fail) d.addErrback(lambda f: f.trap(OSError)) # now check that we keep more recent version d.addCallback(lambda _: m.newTempFile(name, 1024, 1256040206)) d.addCallback(self.completeAndClose, m) d.addCallback(lambda _: m.openCacheFile(name)) d.addCallback(lambda f: setattr(self, "_file", f)) # cache a more recent version d.addCallback(lambda _: m.newTempFile(name, 1024, 1256040208)) d.addCallback(self.completeAndClose, m) # try remove an outdated CachedFile d.addCallback(lambda _: self._file.unlink()) # check file is still there d.addCallback(lambda _: os.stat(path)) return d
def __init__(self, args): props = args['properties'] cacheDir = props.get('cache-dir') cacheSizeInMB = props.get('cache-size') if cacheSizeInMB is not None: cacheSize = cacheSizeInMB * 10**6 # in bytes else: cacheSize = None cleanupEnabled = props.get('cleanup-enabled') cleanupHighWatermark = props.get('cleanup-high-watermark') cleanupLowWatermark = props.get('cleanup-low-watermark') self.virtualHost = props.get('virtual-hostname') self.virtualPort = props.get('virtual-port', DEFAULT_VIRTUAL_PORT) self.virtualPath = props.get('virtual-path', DEFAULT_VIRTUAL_PATH) dnsRefresh = props.get('dns-refresh-period', DEFAULT_DNS_REFRESH) servers = props.get('http-server') compat_servers = props.get('http-server-old') self.stats = cachestats.CacheStatistics() self.cachemgr = cachemanager.CacheManager(self.stats, cacheDir, cacheSize, cleanupEnabled, cleanupHighWatermark, cleanupLowWatermark, self.virtualHost) selector = server_selection.ServerSelector(dnsRefresh) if not (servers or compat_servers): selector.addServer(self.virtualHost, self.virtualPort) else: if compat_servers: # Add the servers specified by name for hostname in compat_servers: if '#' in hostname: hostname, priostr = hostname.split('#', 1) priority = int(priostr) else: priority = DEFAULT_PROXY_PRIORITY if ':' in hostname: hostname, portstr = hostname.split(':', 1) port = int(portstr) else: port = DEFAULT_SERVER_PORT selector.addServer(hostname, port, priority) if servers: # Add the servers specified by compound properties for serverProps in servers: hostname = serverProps.get('hostname') port = serverProps.get('port', DEFAULT_SERVER_PORT) priority = serverProps.get('priority', DEFAULT_PROXY_PRIORITY) selector.addServer(hostname, port, priority) connTimeout = props.get('connection-timeout', DEFAULT_CONN_TIMEOUT) idleTimeout = props.get('idle-timeout', DEFAULT_IDLE_TIMEOUT) client = http_client.StreamRequester(connTimeout, idleTimeout) reqmgr = request_manager.RequestManager(selector, client) cacheTTL = props.get('cache-ttl', DEFAULT_CACHE_TTL) self.strategy = strategy_basic.CachingStrategy(self.cachemgr, reqmgr, cacheTTL) self.resmgr = resource_manager.ResourceManager(self.strategy, self.stats)