def __init__(self, rootDir, resolver):
"""
@type rootDir: str
@param rootDir: root directory of the cache. will be created if it does not exist.
@type resolver: FileResolver
@param resolver: Pluggable component to retrieve (resolve) files.
"""
self.rootDir = rootDir
self.resolver = resolver
self.locksByResource = SynchronizedDict()
requireDir(rootDir)
if not os.path.isdir(rootDir):
raise Exception, 'File cache root dir already exists as a file: %s' % rootDir
def __init__(self, rootDir, resolver):
"""
@type rootDir: str
@param rootDir: root directory of the cache. will be created if it does not exist.
@type resolver: FileResolver
@param resolver: Pluggable component to retrieve (resolve) files.
"""
self.rootDir = rootDir
self.resolver = resolver
self.locksByResource = SynchronizedDict()
requireDir(rootDir)
if not os.path.isdir(rootDir):
raise Exception, 'File cache root dir already exists as a file: %s' % rootDir
class FileCache(object):
"""File cache which uses a FileResolver to populate the cache on-demand"""
def __init__(self, rootDir, resolver):
"""
@type rootDir: str
@param rootDir: root directory of the cache. will be created if it does not exist.
@type resolver: FileResolver
@param resolver: Pluggable component to retrieve (resolve) files.
"""
self.rootDir = rootDir
self.resolver = resolver
self.locksByResource = SynchronizedDict()
requireDir(rootDir)
if not os.path.isdir(rootDir):
raise Exception, 'File cache root dir already exists as a file: %s' % rootDir
def _mapToPath(self, fileUrl):
return os.path.join(self.rootDir, self.resolver.hash(fileUrl))
def contains(self, fileUrl):
return os.path.exists(self._mapToPath(fileUrl))
def get(self, fileUrl):
"""
@return: local path if file resolution was successful, None otherwise
"""
filepath = self._mapToPath(fileUrl)
if not os.path.exists(filepath) or os.path.getsize(filepath) == 0:
log.debug('Cache MISS %s' % safe_str(fileUrl))
self.lockResource(fileUrl)
try:
if not self.contains(fileUrl):
self.resolver.store(fileUrl, filepath)
finally:
self.unlockResource(fileUrl)
# Don't cache zero byte files
if not os.path.exists(filepath):
log.warn('File could not be resolved: %s and not at path: %s' %
(safe_str(fileUrl), filepath))
return None
if os.path.getsize(filepath) == 0:
log.warn('file %s resulted in zero byte file...removing...' %
safe_str(fileUrl))
self.remove(fileUrl)
return None
else:
if log.isEnabledFor(logging.DEBUG):
if hasattr(fileUrl, 'title'):
s = fileUrl.title()
elif hasattr(fileUrl, 'getChannelName'):
s = fileUrl.getChannelName()
else:
s = fileUrl
#log.debug('Cache HIT %s ' % safe_str(s))
return filepath
@sync_instance
def createAndClaimLock(self, resource):
if not self.locksByResource.has_key(resource):
#log.debug('Thread created lock %s' % threading.currentThread().getName())
lock = threading.RLock()
lock.acquire()
self.locksByResource.put(resource, lock)
else:
#log.debug('Thread nearly createdlock %s' % threading.currentThread().getName())
lock = self.locksByResource.get(resource)
lock.acquire()
def lockResource(self, resource):
if self.locksByResource.has_key(resource):
#log.debug('Thread waiting for lock %s' % threading.currentThread().getName())
lock = self.locksByResource.get(resource)
lock.acquire()
else:
self.createAndClaimLock(resource)
def unlockResource(self, resource):
lock = self.locksByResource.get(resource)
#log.debug('lock = %s'%lock)
lock.release()
def remove(self, fileUrl):
filepath = self._mapToPath(fileUrl)
if os.path.exists(filepath):
os.remove(filepath)
def clear(self):
shutil.rmtree(self.rootDir, True)
requireDir(self.rootDir)
self.locksByResource.clear()
class FileCache(object):
"""File cache which uses a FileResolver to populate the cache on-demand"""
def __init__(self, rootDir, resolver):
"""
@type rootDir: str
@param rootDir: root directory of the cache. will be created if it does not exist.
@type resolver: FileResolver
@param resolver: Pluggable component to retrieve (resolve) files.
"""
self.rootDir = rootDir
self.resolver = resolver
self.locksByResource = SynchronizedDict()
requireDir(rootDir)
if not os.path.isdir(rootDir):
raise Exception, 'File cache root dir already exists as a file: %s' % rootDir
def _mapToPath(self, fileUrl):
return os.path.join(self.rootDir, self.resolver.hash(fileUrl))
def contains(self, fileUrl):
return os.path.exists(self._mapToPath(fileUrl))
def get(self, fileUrl):
"""
@return: local path if file resolution was successful, None otherwise
"""
filepath = self._mapToPath(fileUrl)
if not os.path.exists(filepath) or os.path.getsize(filepath) == 0:
log.debug('Cache MISS %s' % safe_str(fileUrl))
self.lockResource(fileUrl)
try:
if not self.contains(fileUrl):
self.resolver.store(fileUrl, filepath)
finally:
self.unlockResource(fileUrl)
# Don't cache zero byte files
if not os.path.exists(filepath):
log.warn('File could not be resolved: %s and not at path: %s' % (safe_str(fileUrl), filepath) )
return None
if os.path.getsize(filepath) == 0:
log.warn('file %s resulted in zero byte file...removing...' % safe_str(fileUrl))
self.remove(fileUrl)
return None
else:
if log.isEnabledFor(logging.DEBUG):
if hasattr(fileUrl, 'title'):
s = fileUrl.title()
elif hasattr(fileUrl, 'getChannelName'):
s = fileUrl.getChannelName()
else:
s = fileUrl
#log.debug('Cache HIT %s ' % safe_str(s))
return filepath
@sync_instance
def createAndClaimLock(self, resource):
if not self.locksByResource.has_key(resource):
#log.debug('Thread created lock %s' % threading.currentThread().getName())
lock = threading.RLock()
lock.acquire()
self.locksByResource.put(resource, lock)
else:
#log.debug('Thread nearly createdlock %s' % threading.currentThread().getName())
lock = self.locksByResource.get(resource)
lock.acquire()
def lockResource(self, resource):
if self.locksByResource.has_key(resource):
#log.debug('Thread waiting for lock %s' % threading.currentThread().getName())
lock = self.locksByResource.get(resource)
lock.acquire()
else:
self.createAndClaimLock(resource)
def unlockResource(self, resource):
lock = self.locksByResource.get(resource)
#log.debug('lock = %s'%lock)
lock.release()
def remove(self, fileUrl):
filepath = self._mapToPath(fileUrl)
if os.path.exists(filepath):
os.remove(filepath)
def clear(self):
shutil.rmtree(self.rootDir, True)
requireDir(self.rootDir)
self.locksByResource.clear()