def __init__(self, args):
props = args['properties']
self._sourceDir = props.get('path')
cacheDir = props.get('cache-dir')
cacheSizeInMB = props.get('cache-size')
if cacheSizeInMB is not None:
cacheSize = cacheSizeInMB * 10**6 # in bytes
else:
cacheSize = None
cleanupEnabled = props.get('cleanup-enabled')
cleanupHighWatermark = props.get('cleanup-high-watermark')
cleanupLowWatermark = props.get('cleanup-low-watermark')
self._sessions = {} # {CopySession: None}
self._index = {} # {path: CopySession}
self.stats = cachestats.CacheStatistics()
self.cache = cachemanager.CacheManager(self.stats, cacheDir, cacheSize,
cleanupEnabled,
cleanupHighWatermark,
cleanupLowWatermark)
common.ensureDir(self._sourceDir, "source")
# Startup copy thread
self._thread = CopyThread(self)
def testConflictMTime(self):
def writeContent(f, content):
f.write(content)
return f
m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True,
0.4, 0.2)
# create a cache for "file' with some old content
d = m.newTempFile("file", 1024, 1256040206)
d.addCallback(writeContent, "old\n")
d.addCallback(lambda f: setattr(self, "_file", f))
# create a second cache for "file' with newer content
d.addCallback(lambda _: m.newTempFile("file", 1024, 1256040207))
d.addCallback(writeContent, "new\n")
# and complete it now
d.addCallback(self.completeAndClose, m)
# then, complete first cache (conflict)
d.addCallback(lambda _: self._file)
d.addCallback(self.completeAndClose, m)
d.addCallback(lambda f: setattr(self, "_file", None))
# check that we get back our "new" content after conflict
d.addCallback(lambda _: m.openCacheFile("file"))
d.addCallback(lambda f: setattr(self, "_file", f))
d.addCallback(lambda _: self._file.readline())
d.addCallback(lambda s: self.assertEqual(s, "new\n"))
d.addCallback(lambda _: self._file.close())
return d
def testHitMiss(self):
m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True,
0.5, 0.2)
# updateCacheUsage ~= 0
d = m.updateCacheUsage()
d.addCallback(self.checkUsage, m, 0)
d.addCallback(lambda _: self.checkCacheMiss(m, "test3"))
d.addCallback(lambda _: self.checkCacheMiss(m, "test4"))
d.addCallback(
lambda _: m.newTempFile("test3", CACHE_SIZE - MAX_PAGE_SIZE))
d.addCallback(lambda f: setattr(self, "_file", f))
d.addCallback(lambda _: self.checkCacheMiss(m, "test3"))
d.addCallback(lambda _: self._file)
d.addCallback(self.completeAndClose, m)
d.addErrback(self.fail)
d.addCallback(lambda _: self.checkCacheHit(m, "test3"))
d.addCallback(lambda _: self.checkCacheMiss(m, "test4"))
d.addCallback(
lambda _: m.newTempFile("test4", CACHE_SIZE - MAX_PAGE_SIZE))
d.addCallback(self.completeAndClose, m)
# The cache cleaning is done in a seperate thread...
d.addCallback(lambda _: time.sleep(1))
d.addCallback(lambda _: self.checkCacheMiss(m, "test3"))
return d
def testLowLevel(self):
def _makeTemp(tag, size, m, name):
t = cachemanager.TempFile(m, name, tag, size)
return t
m = cachemanager.CacheManager(self.stats, self.path,
CACHE_SIZE, True, 0.5, 0.3)
# getIdentifier/Path
self.assertEquals(type(m.getIdentifier("path")), type(""))
self.assertEquals(m.getIdentifier("path"), m.getIdentifier("path"))
self.failIfEquals(m.getIdentifier("path"), m.getIdentifier("path2"))
self.assertEquals(type(m.getCachePath("path")), type(""))
self.assertEquals(type(m.getTempPath("path")), type(""))
# updateCacheUsage ~= 0
d = m.updateCacheUsage()
d.addCallback(self.checkUsage, m, 0)
# allocate&release
d.addCallback(lambda _: m.allocateCacheSpace(100 * 1024))
d.addCallback(self._releaseCacheSpace, m)
# updateCacheUsage ~= 0
d.addCallback(lambda _: m.updateCacheUsage())
d.addCallback(self.checkUsage, m, 0)
# allocate > CACHE_SIZE
d.addCallback(lambda _: m.allocateCacheSpace(CACHE_SIZE * 2))
d.addCallback(lambda c: self.failUnless(c is None))
d.addErrback(self.fail)
# updateCacheUsage ~= 0
d.addCallback(lambda _: m.updateCacheUsage())
d.addCallback(self.checkUsage, m, 0)
# close incomplete TempFile
d.addCallback(lambda _: m.allocateCacheSpace(100 * 1024))
d.addCallback(_makeTemp, 100 * 1024, m, "test")
d.addCallback(lambda t: t.close())
# updateCacheUsage ~= 0
d.addCallback(lambda _: m.updateCacheUsage())
d.addCallback(self.checkUsage, m, 0)
# close complete TempFile
d.addCallback(lambda _: m.allocateCacheSpace(100 * 1024))
d.addCallback(_makeTemp, 100 * 1024, m, "test1")
d.addCallback(self.completeAndClose, m)
# updateCacheUsage ~= 100k
d.addCallback(lambda _: m.updateCacheUsage())
d.addCallback(self.checkUsage, m, 100 * 1024)
return d
def testMTime(self):
m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True,
0.4, 0.2)
d = m.newTempFile("test_mtime", 1024, 1256040206)
d.addCallback(self.completeAndClose, m)
d.addCallback(lambda _: m.openCacheFile("test_mtime"))
d.addCallback(lambda f: f.stat[stat.ST_MTIME])
d.addCallback(lambda t: self.assertEqual(t, 1256040206))
return d
def testCheckCompleted(self):
m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True,
0.4, 0.2)
d = m.newTempFile("test_completed", 5)
d.addCallback(self.checkCompleted)
d.addCallback(lambda _: m.newTempFile("test_incomplete", 5))
d.addCallback(self.checkIncomplete)
return d
def testFileSizeLimit(self):
m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True,
0.4, 0.2)
d = m.newTempFile("test_limit", 5)
d.addCallback(lambda f: f.write("123456"))
d.addCallback(self.fail)
d.addErrback(lambda f: f.trap(IOError))
d.addCallback(lambda _: None)
return d
def testMultiThread(self):
# FIXME: this test can deadlock....
return
m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True,
0.4, 0.2)
dl = []
d = threads.deferToThread(self.fillTestCache2, m)
dl.append(d)
threads.deferToThread(self.fillTestCache2, m)
dl.append(d)
threads.deferToThread(self.fillTestCache2, m)
dl.append(d)
return defer.DeferredList(dl)
def testCacheCleanUp(self):
m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, True,
0.4, 0.2)
self.failIf(m.stats.oncleanup != 0)
d = self.fillTestCache(m, CACHE_SIZE * 4)
# The cache cleaning is done in a seperate thread...
d.addCallback(lambda _: time.sleep(1))
d.addCallback(lambda _: m.updateCacheUsage())
d.addCallback(lambda u: self.failIf(u > CACHE_SIZE / 2))
d.addCallback(lambda _: self.failIf(m.stats.oncleanup < 5))
return d
def testNoCleanUp(self):
m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE * 1.1,
False, 1.0, 0.2)
d = m.newTempFile("FAT FILE", CACHE_SIZE / 2)
d.addCallback(self.completeAndClose, m)
d.addCallback(lambda _: m.newTempFile("FAT FILE TWIN", CACHE_SIZE / 2))
d.addCallback(self.completeAndClose, m)
d.addCallback(lambda _: m.newTempFile("FAT FILE BRO'", CACHE_SIZE / 2))
d.addCallback(lambda t: self.failUnless(t is None))
d.addCallback(lambda _: self.failIf(m.stats.oncleanup != 0))
return d
def testUnlink(self):
m = cachemanager.CacheManager(self.stats, self.path, CACHE_SIZE, False,
0.5, 0.2)
name = "unlink-me"
path = m.getCachePath(name)
d = m.newTempFile(name, 1024, 1256040206)
# TODO: d.addCallback(lambda f: f.complete) is not enough to
# pass the test, weirdo...
d.addCallback(self.completeAndClose, m)
# check that unlink actually works
d.addCallback(lambda _: m.openCacheFile(name))
d.addErrback(lambda _: self.fail)
d.addCallback(lambda c: c.unlink())
d.addCallback(lambda _: os.stat(path))
d.addCallback(self.fail)
d.addErrback(lambda f: f.trap(OSError))
# now check that we keep more recent version
d.addCallback(lambda _: m.newTempFile(name, 1024, 1256040206))
d.addCallback(self.completeAndClose, m)
d.addCallback(lambda _: m.openCacheFile(name))
d.addCallback(lambda f: setattr(self, "_file", f))
# cache a more recent version
d.addCallback(lambda _: m.newTempFile(name, 1024, 1256040208))
d.addCallback(self.completeAndClose, m)
# try remove an outdated CachedFile
d.addCallback(lambda _: self._file.unlink())
# check file is still there
d.addCallback(lambda _: os.stat(path))
return d
def __init__(self, args):
props = args['properties']
cacheDir = props.get('cache-dir')
cacheSizeInMB = props.get('cache-size')
if cacheSizeInMB is not None:
cacheSize = cacheSizeInMB * 10**6 # in bytes
else:
cacheSize = None
cleanupEnabled = props.get('cleanup-enabled')
cleanupHighWatermark = props.get('cleanup-high-watermark')
cleanupLowWatermark = props.get('cleanup-low-watermark')
self.virtualHost = props.get('virtual-hostname')
self.virtualPort = props.get('virtual-port', DEFAULT_VIRTUAL_PORT)
self.virtualPath = props.get('virtual-path', DEFAULT_VIRTUAL_PATH)
dnsRefresh = props.get('dns-refresh-period', DEFAULT_DNS_REFRESH)
servers = props.get('http-server')
compat_servers = props.get('http-server-old')
self.stats = cachestats.CacheStatistics()
self.cachemgr = cachemanager.CacheManager(self.stats, cacheDir,
cacheSize, cleanupEnabled,
cleanupHighWatermark,
cleanupLowWatermark,
self.virtualHost)
selector = server_selection.ServerSelector(dnsRefresh)
if not (servers or compat_servers):
selector.addServer(self.virtualHost, self.virtualPort)
else:
if compat_servers:
# Add the servers specified by name
for hostname in compat_servers:
if '#' in hostname:
hostname, priostr = hostname.split('#', 1)
priority = int(priostr)
else:
priority = DEFAULT_PROXY_PRIORITY
if ':' in hostname:
hostname, portstr = hostname.split(':', 1)
port = int(portstr)
else:
port = DEFAULT_SERVER_PORT
selector.addServer(hostname, port, priority)
if servers:
# Add the servers specified by compound properties
for serverProps in servers:
hostname = serverProps.get('hostname')
port = serverProps.get('port', DEFAULT_SERVER_PORT)
priority = serverProps.get('priority',
DEFAULT_PROXY_PRIORITY)
selector.addServer(hostname, port, priority)
connTimeout = props.get('connection-timeout', DEFAULT_CONN_TIMEOUT)
idleTimeout = props.get('idle-timeout', DEFAULT_IDLE_TIMEOUT)
client = http_client.StreamRequester(connTimeout, idleTimeout)
reqmgr = request_manager.RequestManager(selector, client)
cacheTTL = props.get('cache-ttl', DEFAULT_CACHE_TTL)
self.strategy = strategy_basic.CachingStrategy(self.cachemgr, reqmgr,
cacheTTL)
self.resmgr = resource_manager.ResourceManager(self.strategy,
self.stats)