def __init__(self, args):
props = args['properties']
self._sourceDir = props.get('path')
cacheDir = props.get('cache-dir')
cacheSizeInMB = props.get('cache-size')
if cacheSizeInMB is not None:
cacheSize = cacheSizeInMB * 10**6 # in bytes
else:
cacheSize = None
cleanupEnabled = props.get('cleanup-enabled')
cleanupHighWatermark = props.get('cleanup-high-watermark')
cleanupLowWatermark = props.get('cleanup-low-watermark')
self._sessions = {} # {CopySession: None}
self._index = {} # {path: CopySession}
self.stats = cachestats.CacheStatistics()
self.cache = cachemanager.CacheManager(self.stats, cacheDir, cacheSize,
cleanupEnabled,
cleanupHighWatermark,
cleanupLowWatermark)
common.ensureDir(self._sourceDir, "source")
# Startup copy thread
self._thread = CopyThread(self)
def __init__(self, args):
props = args['properties']
self._sourceDir = props.get('path')
cacheDir = props.get('cache-dir')
cacheSizeInMB = props.get('cache-size')
if cacheSizeInMB is not None:
cacheSize = cacheSizeInMB * 10 ** 6 # in bytes
else:
cacheSize = None
cleanupEnabled = props.get('cleanup-enabled')
cleanupHighWatermark = props.get('cleanup-high-watermark')
cleanupLowWatermark = props.get('cleanup-low-watermark')
self._sessions = {} # {CopySession: None}
self._index = {} # {path: CopySession}
self.stats = cachestats.CacheStatistics()
self.cache = cachemanager.CacheManager(self.stats,
cacheDir, cacheSize,
cleanupEnabled,
cleanupHighWatermark,
cleanupLowWatermark)
common.ensureDir(self._sourceDir, "source")
# Startup copy thread
self._thread = CopyThread(self)
def _getRecentFilenames():
# DSU, or as perl folks call it, a Schwartz Transform
common.ensureDir(configure.registrydir, "registry dir")
for filename in os.listdir(configure.registrydir):
filename = os.path.join(configure.registrydir, filename)
if filename.endswith('.connection'):
yield filename
def _getRecentFilenames():
# DSU, or as perl folks call it, a Schwartz Transform
common.ensureDir(configure.registrydir, "registry dir")
for filename in os.listdir(configure.registrydir):
filename = os.path.join(configure.registrydir, filename)
if filename.endswith('.connection'):
yield filename
def _acquirePidFile(type, name=None):
"""
Open a PID file for writing, using the given process type and
process name for the filename. The returned file can be then passed
to writePidFile after forking.
@rtype: str
@returns: file object, open for writing
"""
ensureDir(configure.rundir, "rundir")
path = _getPidPath(type, name)
return open(path, 'w')
def _acquirePidFile(type, name=None):
"""
Open a PID file for writing, using the given process type and
process name for the filename. The returned file can be then passed
to writePidFile after forking.
@rtype: str
@returns: file object, open for writing
"""
ensureDir(configure.rundir, "rundir")
path = _getPidPath(type, name)
return open(path, 'w')
def _daemonizeHelper(processType, daemonizeTo='/', processName=None):
"""
Daemonize a process, writing log files and PID files to conventional
locations.
@param processType: The process type, for example 'worker'. Used
as the first part of the log file and PID file names.
@type processType: str
@param daemonizeTo: The directory that the daemon should run in.
@type daemonizeTo: str
@param processName: The service name of the process. Used to
disambiguate different instances of the same daemon.
Used as the second part of log file and PID file names.
@type processName: str
"""
ensureDir(configure.logdir, "log dir")
ensureDir(configure.rundir, "run dir")
ensureDir(configure.cachedir, "cache dir")
ensureDir(configure.registrydir, "registry dir")
pid = getPid(processType, processName)
if pid:
raise SystemError(
"A %s service%s is already running with pid %d" % (
processType, processName and ' named %s' % processName or '',
pid))
log.debug(processType, "%s service named '%s' daemonizing",
processType, processName)
if processName:
logPath = os.path.join(configure.logdir,
'%s.%s.log' % (processType, processName))
else:
logPath = os.path.join(configure.logdir,
'%s.log' % (processType, ))
log.debug(processType, 'Further logging will be done to %s', logPath)
pidFile = _acquirePidFile(processType, processName)
# here we daemonize; so we also change our pid
daemonize(stdout=logPath, stderr=logPath, directory=daemonizeTo)
log.debug(processType, 'Started daemon')
# from now on I should keep running until killed, whatever happens
path = writePidFile(processType, processName, file=pidFile)
log.debug(processType, 'written pid file %s', path)
# import inside function so we avoid affecting startup
from twisted.internet import reactor
def _deletePidFile():
log.debug(processType, 'deleting pid file')
deletePidFile(processType, processName)
reactor.addSystemEventTrigger('after', 'shutdown',
_deletePidFile)
def _daemonizeHelper(processType, daemonizeTo='/', processName=None):
"""
Daemonize a process, writing log files and PID files to conventional
locations.
@param processType: The process type, for example 'worker'. Used
as the first part of the log file and PID file names.
@type processType: str
@param daemonizeTo: The directory that the daemon should run in.
@type daemonizeTo: str
@param processName: The service name of the process. Used to
disambiguate different instances of the same daemon.
Used as the second part of log file and PID file names.
@type processName: str
"""
ensureDir(configure.logdir, "log dir")
ensureDir(configure.rundir, "run dir")
ensureDir(configure.cachedir, "cache dir")
ensureDir(configure.registrydir, "registry dir")
pid = getPid(processType, processName)
if pid:
raise SystemError(
"A %s service%s is already running with pid %d" %
(processType, processName and ' named %s' % processName
or '', pid))
log.debug(processType, "%s service named '%s' daemonizing", processType,
processName)
if processName:
logPath = os.path.join(configure.logdir,
'%s.%s.log' % (processType, processName))
else:
logPath = os.path.join(configure.logdir, '%s.log' % (processType, ))
log.debug(processType, 'Further logging will be done to %s', logPath)
pidFile = _acquirePidFile(processType, processName)
# here we daemonize; so we also change our pid
daemonize(stdout=logPath, stderr=logPath, directory=daemonizeTo)
log.debug(processType, 'Started daemon')
# from now on I should keep running until killed, whatever happens
path = writePidFile(processType, processName, file=pidFile)
log.debug(processType, 'written pid file %s', path)
# import inside function so we avoid affecting startup
from twisted.internet import reactor
def _deletePidFile():
log.debug(processType, 'deleting pid file')
deletePidFile(processType, processName)
reactor.addSystemEventTrigger('after', 'shutdown', _deletePidFile)
def __init__(self, stats,
cacheDir = None,
cacheSize = None,
cleanupEnabled = None,
cleanupHighWatermark = None,
cleanupLowWatermark = None,
cacheRealm = None):
if cacheDir is None:
cacheDir = DEFAULT_CACHE_DIR
if cacheSize is None:
cacheSize = DEFAULT_CACHE_SIZE
if cleanupEnabled is None:
cleanupEnabled = DEFAULT_CLEANUP_ENABLED
if cleanupHighWatermark is None:
cleanupHighWatermark = DEFAULT_CLEANUP_HIGH_WATERMARK
if cleanupLowWatermark is None:
cleanupLowWatermark = DEFAULT_CLEANUP_LOW_WATERMARK
self.stats = stats
self._cacheDir = cacheDir
self._cacheSize = cacheSize # in bytes
self._cleanupEnabled = cleanupEnabled
highWatermark = max(0.0, min(1.0, float(cleanupHighWatermark)))
lowWatermark = max(0.0, min(1.0, float(cleanupLowWatermark)))
self._cachePrefix = (cacheRealm and (cacheRealm + ":")) or ""
self._identifiers = {} # {path: identifier}
self.info("Cache Manager initialized")
self.debug("Cache directory: '%s'", self._cacheDir)
self.debug("Cache size: %d bytes", self._cacheSize)
self.debug("Cache cleanup enabled: %s", self._cleanupEnabled)
common.ensureDir(self._cacheDir, "cache")
self._cacheUsage = None
self._cacheUsageLastUpdate = None
self._lastCacheTime = None
self._cacheMaxUsage = self._cacheSize * highWatermark # in bytes
self._cacheMinUsage = self._cacheSize * lowWatermark # in bytes
def writePidFile(type, name=None, file=None):
"""
Write a pid file in the run directory, using the given process type
and process name for the filename.
@rtype: str
@returns: full path to the pid file that was written
"""
# don't shadow builtin file
pidFile = file
if pidFile is None:
ensureDir(configure.rundir, "rundir")
filename = _getPidPath(type, name)
pidFile = open(filename, 'w')
else:
filename = pidFile.name
pidFile.write("%d\n" % (os.getpid(), ))
pidFile.close()
os.chmod(filename, 0644)
return filename
def writePidFile(type, name=None, file=None):
"""
Write a pid file in the run directory, using the given process type
and process name for the filename.
@rtype: str
@returns: full path to the pid file that was written
"""
# don't shadow builtin file
pidFile = file
if pidFile is None:
ensureDir(configure.rundir, "rundir")
filename = _getPidPath(type, name)
pidFile = open(filename, 'w')
else:
filename = pidFile.name
pidFile.write("%d\n" % (os.getpid(), ))
pidFile.close()
os.chmod(filename, 0644)
return filename
def __init__(self, args):
props = args['properties']
self._sourceDir = props.get('path')
self._cacheDir = props.get('cache-dir', "/tmp")
cacheSizeInMB = int(props.get('cache-size', DEFAULT_CACHE_SIZE))
self._cacheSize = cacheSizeInMB * 10 ** 6 # in bytes
self._cleanupEnabled = props.get('cleanup-enabled', True)
highWatermark = props.get('cleanup-high-watermark',
DEFAULT_CLEANUP_HIGH_WATERMARK)
highWatermark = max(0.0, min(1.0, float(highWatermark)))
lowWatermark = props.get('cleanup-low-watermark',
DEFAULT_CLEANUP_LOW_WATERMARK)
lowWatermark = max(0.0, min(1.0, float(lowWatermark)))
self._identifiers = {} # {path: identifier}
self._sessions = {} # {CopySession: None}
self._index = {} # {path: CopySession}
self.info("Cached file provider initialized")
self.debug("Source directory: '%s'", self._sourceDir)
self.debug("Cache directory: '%s'", self._cacheDir)
self.debug("Cache size: %d MB", self._cacheSize)
self.debug("Cache cleanup enabled: %s", self._cleanupEnabled)
common.ensureDir(self._sourceDir, "source")
common.ensureDir(self._cacheDir, "cache")
self._cacheUsage = None # in bytes
self._cacheUsageLastUpdate = None
self._lastCacheTime = None
self._cacheMaxUsage = self._cacheSize * highWatermark # in bytes
self._cacheMinUsage = self._cacheSize * lowWatermark # in bytes
self.stats = cachestats.CacheStatistics()
# Initialize cache usage
self.updateCacheUsage()
# Startup copy thread
self._thread = CopyThread(self)
def testListDir(self):
self.tempdir = tempfile.mkdtemp()
# empty tree
a = os.path.join(self.tempdir, 'A')
common.ensureDir(a, "a description")
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs, [])
# add a non-python file
os.system("touch %s" % os.path.join(a, 'test'))
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs, [])
# add a python file; should now get returned
os.system("touch %s" % os.path.join(a, 'test.py'))
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs, [a])
# add another level
b = os.path.join(self.tempdir, 'B')
b = os.path.join(self.tempdir, 'B')
common.ensureDir(b, "a description")
c = os.path.join(b, 'C')
common.ensureDir(c, "a description")
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs, [a])
# add a non-python file
os.system("touch %s" % os.path.join(c, 'test'))
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs, [a])
# add a python file; should now get returned
os.system("touch %s" % os.path.join(c, 'test.py'))
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs.sort(), [a, c].sort())
# cleanup
os.system("rm -r %s" % self.tempdir)
def testListDir(self):
self.tempdir = tempfile.mkdtemp()
# empty tree
a = os.path.join(self.tempdir, 'A')
common.ensureDir(a, "a description")
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs, [])
# add a non-python file
os.system("touch %s" % os.path.join(a, 'test'))
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs, [])
# add a python file; should now get returned
os.system("touch %s" % os.path.join(a, 'test.py'))
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs, [a])
# add another level
b = os.path.join(self.tempdir, 'B')
b = os.path.join(self.tempdir, 'B')
common.ensureDir(b, "a description")
c = os.path.join(b, 'C')
common.ensureDir(c, "a description")
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs, [a])
# add a non-python file
os.system("touch %s" % os.path.join(c, 'test'))
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs, [a])
# add a python file; should now get returned
os.system("touch %s" % os.path.join(c, 'test.py'))
dirs = package._listDirRecursively(self.tempdir)
self.assertEquals(dirs.sort(), [a, c].sort())
# cleanup
os.system("rm -r %s" % self.tempdir)
def testExisting(self):
self.tempdir = tempfile.mkdtemp()
common.ensureDir(self.tempdir, "a description")
os.system("rm -r %s" % self.tempdir)
def testListPyfile(self):
self.tempdir = tempfile.mkdtemp()
# empty tree
a = os.path.join(self.tempdir, 'A')
common.ensureDir(a, "a description")
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [])
# add a non-python file
os.system("touch %s" % os.path.join(a, 'test'))
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [])
# add a __init__ file
os.system("touch %s" % os.path.join(a, '__init__.py'))
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [])
os.system("touch %s" % os.path.join(a, '__init__.pyc'))
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [])
# add a python file; should now get returned
test1 = os.path.join(a, 'test.py')
os.system("touch %s" % test1)
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [test1])
# add another level
b = os.path.join(self.tempdir, 'B')
common.ensureDir(b, "a description")
c = os.path.join(b, 'C')
common.ensureDir(c, "a description")
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [test1])
# add a non-python file
os.system("touch %s" % os.path.join(c, 'test'))
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [test1])
# add a python file; should now get returned
test2 = os.path.join(c, 'test.py')
os.system("touch %s" % test2)
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs.sort(), [test1, test2].sort())
mods = package.findEndModuleCandidates(self.tempdir, prefix='')
self.assertEquals(mods, ['B.C.test', 'A.test'])
# add a python file but with .c; should now get returned, but
# no new module candidate
test3 = os.path.join(c, 'test.pyc')
os.system("touch %s" % test3)
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs.sort(), [test1, test2, test3].sort())
mods = package.findEndModuleCandidates(self.tempdir, prefix='')
self.assertEquals(mods, ['B.C.test', 'A.test'])
# cleanup
os.system("rm -r %s" % self.tempdir)
def testListPyfile(self):
self.tempdir = tempfile.mkdtemp()
# empty tree
a = os.path.join(self.tempdir, 'A')
common.ensureDir(a, "a description")
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [])
# add a non-python file
os.system("touch %s" % os.path.join(a, 'test'))
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [])
# add a __init__ file
os.system("touch %s" % os.path.join(a, '__init__.py'))
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [])
os.system("touch %s" % os.path.join(a, '__init__.pyc'))
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [])
# add a python file; should now get returned
test1 = os.path.join(a, 'test.py')
os.system("touch %s" % test1)
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [test1])
# add another level
b = os.path.join(self.tempdir, 'B')
common.ensureDir(b, "a description")
c = os.path.join(b, 'C')
common.ensureDir(c, "a description")
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [test1])
# add a non-python file
os.system("touch %s" % os.path.join(c, 'test'))
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs, [test1])
# add a python file; should now get returned
test2 = os.path.join(c, 'test.py')
os.system("touch %s" % test2)
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs.sort(), [test1, test2].sort())
mods = package.findEndModuleCandidates(self.tempdir,
prefix='')
self.assertEquals(mods, ['B.C.test', 'A.test'])
# add a python file but with .c; should now get returned, but
# no new module candidate
test3 = os.path.join(c, 'test.pyc')
os.system("touch %s" % test3)
dirs = package._listPyFileRecursively(self.tempdir)
self.assertEquals(dirs.sort(), [test1, test2, test3].sort())
mods = package.findEndModuleCandidates(self.tempdir,
prefix='')
self.assertEquals(mods, ['B.C.test', 'A.test'])
# cleanup
os.system("rm -r %s" % self.tempdir)
def testExisting(self):
self.tempdir = tempfile.mkdtemp()
common.ensureDir(self.tempdir, "a description")
os.system("rm -r %s" % self.tempdir)
