def lock_createfunc(self, wait = True):
if self.funclock is None:
self.funclock = Synchronizer(identifier =
"databasecontainer/funclock/%s" % self.namespacemanager.namespace,
use_files = True, lock_dir = self.namespacemanager.lock_dir)
return self.funclock.acquire_write_lock(wait)
def test_reentrant(self):
sync1 = Synchronizer(identifier='test', use_files=True, lock_dir='./')
sync2 = Synchronizer(identifier='test', use_files=True, lock_dir='./')
sync1.acquire_write_lock()
sync2.acquire_write_lock()
sync2.release_write_lock()
sync1.release_write_lock()
def lock_createfunc(self, wait=True):
if self.funclock is None:
identifier = 'sqlalchemycontainer/funclock/%s' \
% self.namespacemanager.namespace
self.funclock = Synchronizer(identifier, True,
self.namespacemanager.lock_dir)
return self.funclock.acquire_write_lock(wait)
def __init__(self, namespace, **kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
self.lock = Synchronizer(
identifier="memorycontainer/namespacelock/%s" % self.namespace,
use_files=False)
self.dictionary = MemoryNamespaceManager.namespaces.get(
self.namespace, lambda: {})
class FileContainer(Container):
def do_init(self, **kwargs):
self.funclock = None
def create_namespace(cls, namespace, **kwargs):
return FileNamespaceManager(namespace, **kwargs)
create_namespace = classmethod(create_namespace)
def lock_createfunc(self, wait = True):
if self.funclock is None:
self.funclock = Synchronizer(identifier = "filecontainer/funclock/%s" % self.namespacemanager.namespace, use_files = True, lock_dir = self.namespacemanager.lock_dir)
return self.funclock.acquire_write_lock(wait)
def unlock_createfunc(self): self.funclock.release_write_lock()
def lock_createfunc(self, wait = True):
if self.funclock is None:
self.funclock = Synchronizer(identifier =
"memcachedcontainer/funclock/%s" % self.namespacemanager.namespace,
use_files = True, lock_dir = self.namespacemanager.lock_dir)
return self.funclock.acquire_write_lock(wait)
def __init__(self, namespace, data_dir = None, file_dir = None, lock_dir = None, digest_filenames = True, **kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if file_dir is not None:
self.file_dir = file_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or file_dir is required")
else:
self.file_dir = data_dir + "/container_file"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_file_lock"
util.verify_directory(self.file_dir)
util.verify_directory(self.lock_dir)
self.lock = Synchronizer(identifier=self.namespace, use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root=self.file_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.cache')
self.hash = {}
self.debug("data file %s" % self.file)
def __init__(self,
namespace,
dbmmodule=None,
data_dir=None,
dbm_dir=None,
lock_dir=None,
digest_filenames=True,
**kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if dbm_dir is not None:
self.dbm_dir = dbm_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or dbm_dir is required")
else:
self.dbm_dir = data_dir + "/container_dbm"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_dbm_lock"
if dbmmodule is None:
import anydbm
self.dbmmodule = anydbm
else:
self.dbmmodule = dbmmodule
util.verify_directory(self.dbm_dir)
util.verify_directory(self.lock_dir)
self.dbm = None
self.lock = Synchronizer(identifier=self.namespace,
use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root=self.dbm_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.dbm')
self.debug("data file %s" % self.file)
self._checkfile()
class DatabaseContainer(Container):
def do_init(self, data_dir=None, lock_dir=None, **params):
self.funclock = None
def create_namespace(self, namespace, url, **params):
return DatabaseNamespaceManager(namespace, url, **params)
create_namespace = classmethod(create_namespace)
def lock_createfunc(self, wait = True):
if self.funclock is None:
self.funclock = Synchronizer(identifier =
"databasecontainer/funclock/%s" % self.namespacemanager.namespace,
use_files = True, lock_dir = self.namespacemanager.lock_dir)
return self.funclock.acquire_write_lock(wait)
def unlock_createfunc(self):
self.funclock.release_write_lock()
class SQLAlchemyContainer(Container):
def do_init(self, data_dir=None, lock_dir=None, **kwargs):
self.funclock = None
def create_namespace(self, namespace, bind, table, **kwargs):
return SQLAlchemyNamespaceManager(namespace, bind, table, **kwargs)
create_namespace = classmethod(create_namespace)
def lock_createfunc(self, wait=True):
if self.funclock is None:
identifier = 'sqlalchemycontainer/funclock/%s' \
% self.namespacemanager.namespace
self.funclock = Synchronizer(identifier, True,
self.namespacemanager.lock_dir)
return self.funclock.acquire_write_lock(wait)
def unlock_createfunc(self):
self.funclock.release_write_lock()
class MemcachedContainer(Container):
def do_init(self, data_dir=None, lock_dir=None, **params):
self.funclock = None
def create_namespace(self, namespace, url, **params):
return MemcachedNamespaceManager(namespace, url, **params)
create_namespace = classmethod(create_namespace)
def lock_createfunc(self, wait = True):
if self.funclock is None:
self.funclock = Synchronizer(identifier =
"memcachedcontainer/funclock/%s" % self.namespacemanager.namespace,
use_files = True, lock_dir = self.namespacemanager.lock_dir)
return self.funclock.acquire_write_lock(wait)
def unlock_createfunc(self):
self.funclock.release_write_lock()
class MemoryNamespaceManager(NamespaceManager):
namespaces = util.SyncDict(_threading.Lock(), {})
def __init__(self, namespace, **kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
self.lock = Synchronizer(
identifier="memorycontainer/namespacelock/%s" % self.namespace,
use_files=False)
self.dictionary = MemoryNamespaceManager.namespaces.get(
self.namespace, lambda: {})
def do_acquire_read_lock(self):
self.lock.acquire_read_lock()
def do_release_read_lock(self):
self.lock.release_read_lock()
def do_acquire_write_lock(self, wait=True):
return self.lock.acquire_write_lock(wait)
def do_release_write_lock(self):
self.lock.release_write_lock()
# the open and close methods are totally overridden to eliminate
# the unnecessary "open count" computation involved
def open(self, *args, **kwargs):
pass
def close(self, *args, **kwargs):
pass
def __getitem__(self, key):
return self.dictionary[key]
def __contains__(self, key):
return self.dictionary.__contains__(key)
def has_key(self, key):
return self.dictionary.__contains__(key)
def __setitem__(self, key, value):
self.dictionary[key] = value
def __delitem__(self, key):
del self.dictionary[key]
def do_remove(self):
self.dictionary.clear()
def keys(self):
return self.dictionary.keys()
def __init__(self,
namespace,
data_dir=None,
file_dir=None,
lock_dir=None,
digest_filenames=True,
**kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if file_dir is not None:
self.file_dir = file_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or file_dir is required")
else:
self.file_dir = data_dir + "/container_file"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_file_lock"
util.verify_directory(self.file_dir)
util.verify_directory(self.lock_dir)
self.lock = Synchronizer(identifier=self.namespace,
use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root=self.file_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.cache')
self.hash = {}
self.debug("data file %s" % self.file)
def __init__(self, namespace, dbmmodule = None, data_dir = None, dbm_dir = None, lock_dir = None, digest_filenames = True, **kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if dbm_dir is not None:
self.dbm_dir = dbm_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or dbm_dir is required")
else:
self.dbm_dir = data_dir + "/container_dbm"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_dbm_lock"
if dbmmodule is None:
import anydbm
self.dbmmodule = anydbm
else:
self.dbmmodule = dbmmodule
util.verify_directory(self.dbm_dir)
util.verify_directory(self.lock_dir)
self.dbm = None
self.lock = Synchronizer(identifier=self.namespace, use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root= self.dbm_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.dbm')
self.debug("data file %s" % self.file)
self._checkfile()
class MemoryNamespaceManager(NamespaceManager):
namespaces = util.SyncDict(_threading.Lock(), {})
def __init__(self, namespace, **kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
self.lock = Synchronizer(identifier = "memorycontainer/namespacelock/%s" % self.namespace, use_files = False)
self.dictionary = MemoryNamespaceManager.namespaces.get(self.namespace, lambda: {})
def do_acquire_read_lock(self): self.lock.acquire_read_lock()
def do_release_read_lock(self): self.lock.release_read_lock()
def do_acquire_write_lock(self, wait = True): return self.lock.acquire_write_lock(wait)
def do_release_write_lock(self): self.lock.release_write_lock()
# the open and close methods are totally overridden to eliminate
# the unnecessary "open count" computation involved
def open(self, *args, **kwargs):pass
def close(self, *args, **kwargs):pass
def __getitem__(self, key): return self.dictionary[key]
def __contains__(self, key):
return self.dictionary.__contains__(key)
def has_key(self, key):
return self.dictionary.__contains__(key)
def __setitem__(self, key, value):self.dictionary[key] = value
def __delitem__(self, key):
del self.dictionary[key]
def do_remove(self):
self.dictionary.clear()
def keys(self):
return self.dictionary.keys()
class FileNamespaceManager(NamespaceManager):
def __init__(self,
namespace,
data_dir=None,
file_dir=None,
lock_dir=None,
digest_filenames=True,
**kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if file_dir is not None:
self.file_dir = file_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or file_dir is required")
else:
self.file_dir = data_dir + "/container_file"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_file_lock"
util.verify_directory(self.file_dir)
util.verify_directory(self.lock_dir)
self.lock = Synchronizer(identifier=self.namespace,
use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root=self.file_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.cache')
self.hash = {}
self.debug("data file %s" % self.file)
def file_exists(self, file):
if os.access(file, os.F_OK): return True
else: return False
def do_acquire_read_lock(self):
self.lock.acquire_read_lock()
def do_release_read_lock(self):
self.lock.release_read_lock()
def do_acquire_write_lock(self, wait=True):
return self.lock.acquire_write_lock(wait)
def do_release_write_lock(self):
self.lock.release_write_lock()
def do_open(self, flags):
if self.file_exists(self.file):
fh = open(self.file, 'rb')
try:
self.hash = cPickle.load(fh)
except (IOError, OSError, EOFError, cPickle.PickleError):
pass
fh.close()
self.flags = flags
def do_close(self):
if self.flags is not None and (self.flags == 'c' or self.flags == 'w'):
fh = open(self.file, 'wb')
cPickle.dump(self.hash, fh)
fh.close()
self.flags = None
def do_remove(self):
os.remove(self.file)
self.hash = {}
def __getitem__(self, key):
return self.hash[key]
def __contains__(self, key):
return self.hash.has_key(key)
def __setitem__(self, key, value):
self.hash[key] = value
def __delitem__(self, key):
del self.hash[key]
def keys(self):
return self.hash.keys()
class DBMNamespaceManager(NamespaceManager):
def __init__(self,
namespace,
dbmmodule=None,
data_dir=None,
dbm_dir=None,
lock_dir=None,
digest_filenames=True,
**kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if dbm_dir is not None:
self.dbm_dir = dbm_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or dbm_dir is required")
else:
self.dbm_dir = data_dir + "/container_dbm"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_dbm_lock"
if dbmmodule is None:
import anydbm
self.dbmmodule = anydbm
else:
self.dbmmodule = dbmmodule
util.verify_directory(self.dbm_dir)
util.verify_directory(self.lock_dir)
self.dbm = None
self.lock = Synchronizer(identifier=self.namespace,
use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root=self.dbm_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.dbm')
self.debug("data file %s" % self.file)
self._checkfile()
def file_exists(self, file):
if os.access(file, os.F_OK): return True
else:
for ext in ('db', 'dat', 'pag', 'dir'):
if os.access(file + os.extsep + ext, os.F_OK):
return True
return False
def _checkfile(self):
if not self.file_exists(self.file):
g = self.dbmmodule.open(self.file, 'c')
g.close()
def get_filenames(self):
list = []
if os.access(self.file, os.F_OK):
list.append(self.file)
for ext in ('pag', 'dir', 'db', 'dat'):
if os.access(self.file + os.extsep + ext, os.F_OK):
list.append(self.file + os.extsep + ext)
return list
def do_acquire_read_lock(self):
self.lock.acquire_read_lock()
def do_release_read_lock(self):
self.lock.release_read_lock()
def do_acquire_write_lock(self, wait=True):
return self.lock.acquire_write_lock(wait)
def do_release_write_lock(self):
self.lock.release_write_lock()
def do_open(self, flags):
# caution: apparently gdbm handles arent threadsafe, they
# are using flock(), and i would rather not have knowledge
# of the "unlock" 'u' option just for that one dbm module.
# therefore, neither is an individual instance of
# this namespacemanager (of course, multiple nsm's
# can exist for each thread).
self.debug("opening dbm file %s" % self.file)
try:
self.dbm = self.dbmmodule.open(self.file, flags)
except:
self._checkfile()
self.dbm = self.dbmmodule.open(self.file, flags)
def do_close(self):
if self.dbm is not None:
self.debug("closing dbm file %s" % self.file)
self.dbm.close()
def do_remove(self):
for f in self.get_filenames():
os.remove(f)
def __getitem__(self, key):
return cPickle.loads(self.dbm[key])
def __contains__(self, key):
return self.dbm.has_key(key)
def __setitem__(self, key, value):
self.dbm[key] = cPickle.dumps(value)
def __delitem__(self, key):
del self.dbm[key]
def keys(self):
return self.dbm.keys()
def __init__(self, namespace, **kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
self.lock = Synchronizer(identifier = "memorycontainer/namespacelock/%s" % self.namespace, use_files = False)
self.dictionary = MemoryNamespaceManager.namespaces.get(self.namespace, lambda: {})
class DBMNamespaceManager(NamespaceManager):
def __init__(self, namespace, dbmmodule = None, data_dir = None, dbm_dir = None, lock_dir = None, digest_filenames = True, **kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if dbm_dir is not None:
self.dbm_dir = dbm_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or dbm_dir is required")
else:
self.dbm_dir = data_dir + "/container_dbm"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_dbm_lock"
if dbmmodule is None:
self.dbmmodule = anydbm
else:
self.dbmmodule = dbmmodule
util.verify_directory(self.dbm_dir)
util.verify_directory(self.lock_dir)
self.dbm = None
self.lock = Synchronizer(identifier=self.namespace, use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root= self.dbm_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.dbm')
self.debug("data file %s" % self.file)
self._checkfile()
def file_exists(self, file):
if os.access(file, os.F_OK): return True
else:
for ext in ('db', 'dat', 'pag', 'dir'):
if os.access(file + os.extsep + ext, os.F_OK):
return True
return False
def _checkfile(self):
if not self.file_exists(self.file):
g = self.dbmmodule.open(self.file, 'c')
g.close()
def get_filenames(self):
list = []
if os.access(self.file, os.F_OK):
list.append(self.file)
for ext in ('pag', 'dir', 'db', 'dat'):
if os.access(self.file + os.extsep + ext, os.F_OK):
list.append(self.file + os.extsep + ext)
return list
def do_acquire_read_lock(self):
self.lock.acquire_read_lock()
def do_release_read_lock(self):
self.lock.release_read_lock()
def do_acquire_write_lock(self, wait = True):
return self.lock.acquire_write_lock(wait)
def do_release_write_lock(self):
self.lock.release_write_lock()
def do_open(self, flags):
# caution: apparently gdbm handles arent threadsafe, they
# are using flock(), and i would rather not have knowledge
# of the "unlock" 'u' option just for that one dbm module.
# therefore, neither is an individual instance of
# this namespacemanager (of course, multiple nsm's
# can exist for each thread).
self.debug("opening dbm file %s" % self.file)
try:
self.dbm = self.dbmmodule.open(self.file, flags)
except:
self._checkfile()
self.dbm = self.dbmmodule.open(self.file, flags)
def do_close(self):
if self.dbm is not None:
self.debug("closing dbm file %s" % self.file)
self.dbm.close()
def do_remove(self):
for f in self.get_filenames():
os.remove(f)
def __getitem__(self, key):
return cPickle.loads(self.dbm[key])
def __contains__(self, key):
return self.dbm.has_key(key)
def __setitem__(self, key, value):
self.dbm[key] = cPickle.dumps(value)
def __delitem__(self, key):
del self.dbm[key]
def keys(self):
return self.dbm.keys()
class FileNamespaceManager(NamespaceManager):
def __init__(self, namespace, data_dir = None, file_dir = None, lock_dir = None, digest_filenames = True, **kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if file_dir is not None:
self.file_dir = file_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or file_dir is required")
else:
self.file_dir = data_dir + "/container_file"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_file_lock"
util.verify_directory(self.file_dir)
util.verify_directory(self.lock_dir)
self.lock = Synchronizer(identifier=self.namespace, use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root=self.file_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.cache')
self.hash = {}
self.debug("data file %s" % self.file)
def file_exists(self, file):
if os.access(file, os.F_OK): return True
else: return False
def do_acquire_read_lock(self):
self.lock.acquire_read_lock()
def do_release_read_lock(self):
self.lock.release_read_lock()
def do_acquire_write_lock(self, wait = True):
return self.lock.acquire_write_lock(wait)
def do_release_write_lock(self):
self.lock.release_write_lock()
def do_open(self, flags):
if self.file_exists(self.file):
fh = open(self.file, 'rb')
try:
self.hash = cPickle.load(fh)
except (IOError, OSError, EOFError, cPickle.PickleError):
pass
fh.close()
self.flags = flags
def do_close(self):
if self.flags is not None and (self.flags == 'c' or self.flags == 'w'):
fh = open(self.file, 'wb')
cPickle.dump(self.hash, fh)
fh.close()
self.flags = None
def do_remove(self):
os.remove(self.file)
self.hash = {}
def __getitem__(self, key):
return self.hash[key]
def __contains__(self, key):
return self.hash.has_key(key)
def __setitem__(self, key, value):
self.hash[key] = value
def __delitem__(self, key):
del self.hash[key]
def keys(self):
return self.hash.keys()