def listDirectory(self, path):
try:
return self._memoryCache.listDirectory(path)
except memory_cache.MemoryCacheNotValid, e:
if loclogger.debug:
DEBUG("Memory cache is not valid")
return self._diskCache.listDirectory(path)
def walk(self):
dirs, files, links = [], [], []
dirpath = '/'.join([self.rootpath, self.relpath])
for entry in os.listdir(dirpath):
if loclogger.debug:
DEBUG(entry)
try:
full_path = os.sep.join([dirpath, entry])
st = os.lstat(full_path)
st_mode = st.st_mode
if stat.S_ISREG(st_mode):
files.append(entry)
elif stat.S_ISDIR(st_mode):
dirs.append(entry)
elif stat.S_ISLNK(st_mode):
links.append(entry)
else:
ERROR("unsupported type of file %s" % full_path)
# TODO: check if we don't need to slightly modify
# st struct (because of some i-node info)
self.memoryCache.cacheAttributes(os.sep.join([self.relpath, entry]), st)
except OSError:
ERROR("cannot stat: %s" % full_path)
return dirs, files, links
def getAttributes(self, path, pathToCache=False):
memstat = self._memoryCache.getAttributes(path)
if not memstat:
if loclogger.debug:
DEBUG("Checking cache directory for %s" % path)
st = self._getAttributesFromDiskCache(path)
self._memoryCache.cacheAttributes(path, st)
memstat = self._memoryCache.getAttributes(path)
return memstat.stat
def _cacheDirectory(self, path):
sourcePath = self._pathFactory.createAbsoluteSourcePath(path)
pathToCache = self._pathFactory.createPathToDiskCache(path)
try:
os.mkdir(pathToCache)
except OSError, e:
parent_path = os.path.dirname(path)
if e.errno == errno.ENOENT and '/' <> parent_path:
self._cacheDirectory(parent_path)
os.mkdir(pathToCache)
if loclogger.debug:
DEBUG("most likely directory %s already exists" % pathToCache)
def _getAttributes(self, path):
node = self._get_node(path)
if node:
if node.parent is None: # i.e. root
if node.stat is None:
return None
if time.time() - node.timestamp < self._cache_lifetime:
if loclogger.debug:
DEBUG("time now: " + str(time.time()) +
", memory cache lifetime: " +
str(self._cache_lifetime) + ", node timestamp is: " +
str(node.timestamp))
return node
else:
self._remove_subtree(node)
return None
else:
if loclogger.debug:
DEBUG("NO CACHE ENTRY FOR %s" % path)
return None
def _remove_subtree(self, node):
# NOTE: root is currently not removeable
# method is not thread safe
if node.parent:
# search for key with given value
basename = [
key for key, value in node.parent.children.iteritems()
if value == node
][0]
DEBUG("key to remove: " + key)
del node.parent.children[basename]
else:
node.stat = None # FIXME: ugly workaround - deinitialization of root
def _getAttributesFromDiskCache(self, path):
try:
st = self._diskCache.getAttributes(path)
if (not st is None and stat.S_ISDIR(st.st_mode)):
self._diskCache.cacheDirectory(path)
return st
except disk_cache.ParentDirNotCached, e:
if loclogger.debug:
DEBUG("Needs to cache directory: %s" % path)
self._diskCache.cacheDirectory(os.path.dirname(path))
return self._getAttributesFromDiskCache(path)
def cacheAttributes(self, path, st):
if loclogger.debug:
DEBUG("DummyMemcache.cacheAttributes(%s, %s)" % (path, st))
if e.errno == errno.ENOENT and '/' <> parent_path:
self._cacheDirectory(parent_path)
os.mkdir(pathToCache)
if loclogger.debug:
DEBUG("most likely directory %s already exists" % pathToCache)
cacheWalker = self._createCachedDirWalker(pathToCache)
sourceDirWalker = self._createDirectoryWalker(self._config.source_dir, path)
sourceDirWalker.initialize()
notCachedFiles = list(set(sourceDirWalker.files) - set(cacheWalker.files))
notCachedDirectories = list(set(sourceDirWalker.dirs) - set(cacheWalker.dirs))
notCachedLinks = list(set(sourceDirWalker.links) - set(cacheWalker.links))
if loclogger.debug:
DEBUG("REMOTE PATH: %s" % sourcePath)
DEBUG("Files to be cached: %s" % notCachedFiles)
DEBUG("Directories to be cached: %s" % notCachedDirectories)
DEBUG("Links to be cached: %s" % notCachedLinks)
for filename in notCachedFiles:
fileMarker = self._pathFactory.createPathToDiskCacheFileMarker(os.path.join(path, filename))
os.symlink(filename, fileMarker)
for dirname in notCachedDirectories:
directoryMarker = self._pathFactory.createPathToDiskCacheDirectoryMarker(os.path.join(path, dirname))
os.mkdir(directoryMarker)
for link in notCachedLinks:
link_target = os.readlink(os.sep.join([sourcePath, link]))
os.symlink(link_target, os.sep.join([pathToCache, link]))
if not fileExists is None:
return fileExists
return self._diskCache.exists(path)
def _getAttributesFromDiskCache(self, path):
try:
st = self._diskCache.getAttributes(path)
if (not st is None and stat.S_ISDIR(st.st_mode)):
self._diskCache.cacheDirectory(path)
return st
except disk_cache.ParentDirNotCached, e:
if loclogger.debug:
DEBUG("Needs to cache directory: %s" % path)
self._diskCache.cacheDirectory(os.path.dirname(path))
return self._getAttributesFromDiskCache(path)
except disk_cache.OriginFileNotExists, e:
if loclogger.debug:
DEBUG("File doesn't exist %s" % path)
return None
def _prepare_directories(self):
dir = self._cfg.cache_root_dir
if not os.path.exists(dir):
os.makedirs(dir)
import mocks.time_mock # file available in tests directory
time = mocks.time_mock.ModuleInterface()
memory_cache.time = time
time_stubbed = True
except Exception, e:
# standard environment
import time as time_module
time = time_module
memory_cache.time = time_module
time_stubbed = False
try:
if time_stubbed:
time.timeMock.initialize()
main()
except Exception, e:
print(str(e))
finally:
if time_stubbed:
time.timeMock.dispose()
DEBUG("time stubbed: " + str(time_stubbed))
INFO("File system unmounted")
else:
import time as time_module
time = time_module