def __init__(self, args): props = args['properties'] self._sourceDir = props.get('path') cacheDir = props.get('cache-dir') cacheSizeInMB = props.get('cache-size') if cacheSizeInMB is not None: cacheSize = cacheSizeInMB * 10**6 # in bytes else: cacheSize = None cleanupEnabled = props.get('cleanup-enabled') cleanupHighWatermark = props.get('cleanup-high-watermark') cleanupLowWatermark = props.get('cleanup-low-watermark') self._sessions = {} # {CopySession: None} self._index = {} # {path: CopySession} self.stats = cachestats.CacheStatistics() self.cache = cachemanager.CacheManager(self.stats, cacheDir, cacheSize, cleanupEnabled, cleanupHighWatermark, cleanupLowWatermark) common.ensureDir(self._sourceDir, "source") # Startup copy thread self._thread = CopyThread(self)
def __init__(self, args): props = args['properties'] self._sourceDir = props.get('path') cacheDir = props.get('cache-dir') cacheSizeInMB = props.get('cache-size') if cacheSizeInMB is not None: cacheSize = cacheSizeInMB * 10 ** 6 # in bytes else: cacheSize = None cleanupEnabled = props.get('cleanup-enabled') cleanupHighWatermark = props.get('cleanup-high-watermark') cleanupLowWatermark = props.get('cleanup-low-watermark') self._sessions = {} # {CopySession: None} self._index = {} # {path: CopySession} self.stats = cachestats.CacheStatistics() self.cache = cachemanager.CacheManager(self.stats, cacheDir, cacheSize, cleanupEnabled, cleanupHighWatermark, cleanupLowWatermark) common.ensureDir(self._sourceDir, "source") # Startup copy thread self._thread = CopyThread(self)
def _getRecentFilenames(): # DSU, or as perl folks call it, a Schwartz Transform common.ensureDir(configure.registrydir, "registry dir") for filename in os.listdir(configure.registrydir): filename = os.path.join(configure.registrydir, filename) if filename.endswith('.connection'): yield filename
def _acquirePidFile(type, name=None): """ Open a PID file for writing, using the given process type and process name for the filename. The returned file can be then passed to writePidFile after forking. @rtype: str @returns: file object, open for writing """ ensureDir(configure.rundir, "rundir") path = _getPidPath(type, name) return open(path, 'w')
def _daemonizeHelper(processType, daemonizeTo='/', processName=None): """ Daemonize a process, writing log files and PID files to conventional locations. @param processType: The process type, for example 'worker'. Used as the first part of the log file and PID file names. @type processType: str @param daemonizeTo: The directory that the daemon should run in. @type daemonizeTo: str @param processName: The service name of the process. Used to disambiguate different instances of the same daemon. Used as the second part of log file and PID file names. @type processName: str """ ensureDir(configure.logdir, "log dir") ensureDir(configure.rundir, "run dir") ensureDir(configure.cachedir, "cache dir") ensureDir(configure.registrydir, "registry dir") pid = getPid(processType, processName) if pid: raise SystemError( "A %s service%s is already running with pid %d" % ( processType, processName and ' named %s' % processName or '', pid)) log.debug(processType, "%s service named '%s' daemonizing", processType, processName) if processName: logPath = os.path.join(configure.logdir, '%s.%s.log' % (processType, processName)) else: logPath = os.path.join(configure.logdir, '%s.log' % (processType, )) log.debug(processType, 'Further logging will be done to %s', logPath) pidFile = _acquirePidFile(processType, processName) # here we daemonize; so we also change our pid daemonize(stdout=logPath, stderr=logPath, directory=daemonizeTo) log.debug(processType, 'Started daemon') # from now on I should keep running until killed, whatever happens path = writePidFile(processType, processName, file=pidFile) log.debug(processType, 'written pid file %s', path) # import inside function so we avoid affecting startup from twisted.internet import reactor def _deletePidFile(): log.debug(processType, 'deleting pid file') deletePidFile(processType, processName) reactor.addSystemEventTrigger('after', 'shutdown', _deletePidFile)
def _daemonizeHelper(processType, daemonizeTo='/', processName=None): """ Daemonize a process, writing log files and PID files to conventional locations. @param processType: The process type, for example 'worker'. Used as the first part of the log file and PID file names. @type processType: str @param daemonizeTo: The directory that the daemon should run in. @type daemonizeTo: str @param processName: The service name of the process. Used to disambiguate different instances of the same daemon. Used as the second part of log file and PID file names. @type processName: str """ ensureDir(configure.logdir, "log dir") ensureDir(configure.rundir, "run dir") ensureDir(configure.cachedir, "cache dir") ensureDir(configure.registrydir, "registry dir") pid = getPid(processType, processName) if pid: raise SystemError( "A %s service%s is already running with pid %d" % (processType, processName and ' named %s' % processName or '', pid)) log.debug(processType, "%s service named '%s' daemonizing", processType, processName) if processName: logPath = os.path.join(configure.logdir, '%s.%s.log' % (processType, processName)) else: logPath = os.path.join(configure.logdir, '%s.log' % (processType, )) log.debug(processType, 'Further logging will be done to %s', logPath) pidFile = _acquirePidFile(processType, processName) # here we daemonize; so we also change our pid daemonize(stdout=logPath, stderr=logPath, directory=daemonizeTo) log.debug(processType, 'Started daemon') # from now on I should keep running until killed, whatever happens path = writePidFile(processType, processName, file=pidFile) log.debug(processType, 'written pid file %s', path) # import inside function so we avoid affecting startup from twisted.internet import reactor def _deletePidFile(): log.debug(processType, 'deleting pid file') deletePidFile(processType, processName) reactor.addSystemEventTrigger('after', 'shutdown', _deletePidFile)
def __init__(self, stats, cacheDir = None, cacheSize = None, cleanupEnabled = None, cleanupHighWatermark = None, cleanupLowWatermark = None, cacheRealm = None): if cacheDir is None: cacheDir = DEFAULT_CACHE_DIR if cacheSize is None: cacheSize = DEFAULT_CACHE_SIZE if cleanupEnabled is None: cleanupEnabled = DEFAULT_CLEANUP_ENABLED if cleanupHighWatermark is None: cleanupHighWatermark = DEFAULT_CLEANUP_HIGH_WATERMARK if cleanupLowWatermark is None: cleanupLowWatermark = DEFAULT_CLEANUP_LOW_WATERMARK self.stats = stats self._cacheDir = cacheDir self._cacheSize = cacheSize # in bytes self._cleanupEnabled = cleanupEnabled highWatermark = max(0.0, min(1.0, float(cleanupHighWatermark))) lowWatermark = max(0.0, min(1.0, float(cleanupLowWatermark))) self._cachePrefix = (cacheRealm and (cacheRealm + ":")) or "" self._identifiers = {} # {path: identifier} self.info("Cache Manager initialized") self.debug("Cache directory: '%s'", self._cacheDir) self.debug("Cache size: %d bytes", self._cacheSize) self.debug("Cache cleanup enabled: %s", self._cleanupEnabled) common.ensureDir(self._cacheDir, "cache") self._cacheUsage = None self._cacheUsageLastUpdate = None self._lastCacheTime = None self._cacheMaxUsage = self._cacheSize * highWatermark # in bytes self._cacheMinUsage = self._cacheSize * lowWatermark # in bytes
def writePidFile(type, name=None, file=None): """ Write a pid file in the run directory, using the given process type and process name for the filename. @rtype: str @returns: full path to the pid file that was written """ # don't shadow builtin file pidFile = file if pidFile is None: ensureDir(configure.rundir, "rundir") filename = _getPidPath(type, name) pidFile = open(filename, 'w') else: filename = pidFile.name pidFile.write("%d\n" % (os.getpid(), )) pidFile.close() os.chmod(filename, 0644) return filename
def __init__(self, args): props = args['properties'] self._sourceDir = props.get('path') self._cacheDir = props.get('cache-dir', "/tmp") cacheSizeInMB = int(props.get('cache-size', DEFAULT_CACHE_SIZE)) self._cacheSize = cacheSizeInMB * 10 ** 6 # in bytes self._cleanupEnabled = props.get('cleanup-enabled', True) highWatermark = props.get('cleanup-high-watermark', DEFAULT_CLEANUP_HIGH_WATERMARK) highWatermark = max(0.0, min(1.0, float(highWatermark))) lowWatermark = props.get('cleanup-low-watermark', DEFAULT_CLEANUP_LOW_WATERMARK) lowWatermark = max(0.0, min(1.0, float(lowWatermark))) self._identifiers = {} # {path: identifier} self._sessions = {} # {CopySession: None} self._index = {} # {path: CopySession} self.info("Cached file provider initialized") self.debug("Source directory: '%s'", self._sourceDir) self.debug("Cache directory: '%s'", self._cacheDir) self.debug("Cache size: %d MB", self._cacheSize) self.debug("Cache cleanup enabled: %s", self._cleanupEnabled) common.ensureDir(self._sourceDir, "source") common.ensureDir(self._cacheDir, "cache") self._cacheUsage = None # in bytes self._cacheUsageLastUpdate = None self._lastCacheTime = None self._cacheMaxUsage = self._cacheSize * highWatermark # in bytes self._cacheMinUsage = self._cacheSize * lowWatermark # in bytes self.stats = cachestats.CacheStatistics() # Initialize cache usage self.updateCacheUsage() # Startup copy thread self._thread = CopyThread(self)
def testListDir(self): self.tempdir = tempfile.mkdtemp() # empty tree a = os.path.join(self.tempdir, 'A') common.ensureDir(a, "a description") dirs = package._listDirRecursively(self.tempdir) self.assertEquals(dirs, []) # add a non-python file os.system("touch %s" % os.path.join(a, 'test')) dirs = package._listDirRecursively(self.tempdir) self.assertEquals(dirs, []) # add a python file; should now get returned os.system("touch %s" % os.path.join(a, 'test.py')) dirs = package._listDirRecursively(self.tempdir) self.assertEquals(dirs, [a]) # add another level b = os.path.join(self.tempdir, 'B') b = os.path.join(self.tempdir, 'B') common.ensureDir(b, "a description") c = os.path.join(b, 'C') common.ensureDir(c, "a description") dirs = package._listDirRecursively(self.tempdir) self.assertEquals(dirs, [a]) # add a non-python file os.system("touch %s" % os.path.join(c, 'test')) dirs = package._listDirRecursively(self.tempdir) self.assertEquals(dirs, [a]) # add a python file; should now get returned os.system("touch %s" % os.path.join(c, 'test.py')) dirs = package._listDirRecursively(self.tempdir) self.assertEquals(dirs.sort(), [a, c].sort()) # cleanup os.system("rm -r %s" % self.tempdir)
def testExisting(self): self.tempdir = tempfile.mkdtemp() common.ensureDir(self.tempdir, "a description") os.system("rm -r %s" % self.tempdir)
def testListPyfile(self): self.tempdir = tempfile.mkdtemp() # empty tree a = os.path.join(self.tempdir, 'A') common.ensureDir(a, "a description") dirs = package._listPyFileRecursively(self.tempdir) self.assertEquals(dirs, []) # add a non-python file os.system("touch %s" % os.path.join(a, 'test')) dirs = package._listPyFileRecursively(self.tempdir) self.assertEquals(dirs, []) # add a __init__ file os.system("touch %s" % os.path.join(a, '__init__.py')) dirs = package._listPyFileRecursively(self.tempdir) self.assertEquals(dirs, []) os.system("touch %s" % os.path.join(a, '__init__.pyc')) dirs = package._listPyFileRecursively(self.tempdir) self.assertEquals(dirs, []) # add a python file; should now get returned test1 = os.path.join(a, 'test.py') os.system("touch %s" % test1) dirs = package._listPyFileRecursively(self.tempdir) self.assertEquals(dirs, [test1]) # add another level b = os.path.join(self.tempdir, 'B') common.ensureDir(b, "a description") c = os.path.join(b, 'C') common.ensureDir(c, "a description") dirs = package._listPyFileRecursively(self.tempdir) self.assertEquals(dirs, [test1]) # add a non-python file os.system("touch %s" % os.path.join(c, 'test')) dirs = package._listPyFileRecursively(self.tempdir) self.assertEquals(dirs, [test1]) # add a python file; should now get returned test2 = os.path.join(c, 'test.py') os.system("touch %s" % test2) dirs = package._listPyFileRecursively(self.tempdir) self.assertEquals(dirs.sort(), [test1, test2].sort()) mods = package.findEndModuleCandidates(self.tempdir, prefix='') self.assertEquals(mods, ['B.C.test', 'A.test']) # add a python file but with .c; should now get returned, but # no new module candidate test3 = os.path.join(c, 'test.pyc') os.system("touch %s" % test3) dirs = package._listPyFileRecursively(self.tempdir) self.assertEquals(dirs.sort(), [test1, test2, test3].sort()) mods = package.findEndModuleCandidates(self.tempdir, prefix='') self.assertEquals(mods, ['B.C.test', 'A.test']) # cleanup os.system("rm -r %s" % self.tempdir)