def __init__(
self,
namespace,
url,
data_dir=None,
lock_dir=None,
hkey='sessions',
**params):
super().__init__(namespace)
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
else:
self.lock_dir = None
if self.lock_dir:
verify_directory(self.lock_dir)
self.hkey = hkey
host, port_db = url.split(':', 1)
port, db = (int(i) for i in port_db.split('/', 1))
self.db = Redis(host=host, port=port, db=db)
log.debug('sessions redis backend setuped: namespace={}'.format(namespace))
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
else:
self.lock_dir = None
if self.lock_dir:
verify_directory(self.lock_dir)
self.url = url
# parse the url properly using urlparse
url = urlparse.urlparse(url)
conn_params = {}
parts = url.path.split('?', 1)
path = parts[0]
if len(parts) > 1:
conn_params = dict(p.split('=', 1) for p in parts[1].split('&'))
self.host = url.hostname
self.port = url.port
self.username = url.username
self.password = url.password
self.scheme = url.scheme
self.open_connection(self.host, self.port, **conn_params)
def __init__(self, namespace, dbmmodule=None, data_dir=None,
dbm_dir=None, lock_dir=None, digest_filenames=True, **kwargs):
self.digest_filenames = digest_filenames
if not dbm_dir and not data_dir:
raise MissingCacheParameter("data_dir or dbm_dir is required")
elif dbm_dir:
self.dbm_dir = dbm_dir
else:
self.dbm_dir = data_dir + "/container_dbm"
util.verify_directory(self.dbm_dir)
if not lock_dir and not data_dir:
raise MissingCacheParameter("data_dir or lock_dir is required")
elif lock_dir:
self.lock_dir = lock_dir
else:
self.lock_dir = data_dir + "/container_dbm_lock"
util.verify_directory(self.lock_dir)
self.dbmmodule = dbmmodule or anydbm
self.dbm = None
OpenResourceNamespaceManager.__init__(self, namespace)
self.file = util.encoded_path(root= self.dbm_dir,
identifiers=[self.namespace],
extension='.dbm',
digest_filenames=self.digest_filenames)
debug("data file %s", self.file)
self._checkfile()
def __init__(self, namespace, data_dir=None, file_dir=None, lock_dir=None,
digest_filenames=True, **kwargs):
self.digest_filenames = digest_filenames
if not file_dir and not data_dir:
raise MissingCacheParameter("data_dir or file_dir is required")
elif file_dir:
self.file_dir = file_dir
else:
self.file_dir = data_dir + "/container_file"
util.verify_directory(self.file_dir)
if not lock_dir and not data_dir:
raise MissingCacheParameter("data_dir or lock_dir is required")
elif lock_dir:
self.lock_dir = lock_dir
else:
self.lock_dir = data_dir + "/container_file_lock"
util.verify_directory(self.lock_dir)
OpenResourceNamespaceManager.__init__(self, namespace)
self.file = util.encoded_path(root=self.file_dir,
identifiers=[self.namespace],
extension='.cache',
digest_filenames=self.digest_filenames)
self.hash = {}
debug("data file %s", self.file)
def __init__(
self,
namespace,
dsn='localhost:6379/0',
data_dir=None,
lock_dir=None,
hkey_prefix='session',
ttl=DEFAULT_TTL,
**params):
super().__init__(namespace)
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
else:
self.lock_dir = None
if self.lock_dir:
verify_directory(self.lock_dir)
host, port_db = dsn.split(':', 1)
port, db_num = (int(i) for i in port_db.split('/', 1))
self.db = Redis(host=host, port=port, db=db_num)
self.hkey = hkey = ':'.join((hkey_prefix, self.namespace))
self.ttl = int(ttl)
log.debug("session setuped: %s", hkey)
def __init__(self,
namespace,
url,
memcache_module='auto',
data_dir=None,
lock_dir=None,
**kw):
NamespaceManager.__init__(self, namespace)
_memcache_module = _client_libs[memcache_module]
if not url:
raise MissingCacheParameter("url is required")
self.lock_dir = None
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mcd_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
# Check for pylibmc namespace manager, in which case client will be
# instantiated by subclass __init__, to handle behavior passing to the
# pylibmc client
if not _is_configured_for_pylibmc(memcache_module, _memcache_module):
self.mc = MemcachedNamespaceManager.clients.get(
(memcache_module, url), _memcache_module.Client,
url.split(';'))
def __init__(self,
namespace,
bind,
table,
data_dir=None,
lock_dir=None,
**kwargs):
"""Create a namespace manager for use with a database table via
SQLAlchemy.
``bind``
SQLAlchemy ``Engine`` or ``Connection`` object
``table``
SQLAlchemy ``Table`` object in which to store namespace data.
This should usually be something created by ``make_cache_table``.
"""
OpenResourceNamespaceManager.__init__(self, namespace)
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_db_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
self.bind = self.__class__.binds.get(str(bind.url), lambda: bind)
self.table = self.__class__.tables.get(
'%s:%s' % (bind.url, table.name), lambda: table)
self.hash = {}
self._is_new = False
self.loaded = False
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
expiretime=None,
**params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
if hasattr(self, 'lock_dir'):
verify_directory(self.lock_dir)
# Specify the serializer to use (pickle or json?)
self.serializer = params.pop('serializer', 'pickle')
self._expiretime = int(expiretime) if expiretime else None
conn_params = {}
parts = url.split('?', 1)
url = parts[0]
if len(parts) > 1:
conn_params = dict(p.split('=', 1) for p in parts[1].split('&'))
host, port = url.split(':', 1)
self.open_connection(host, int(port), **conn_params)
def __init__(self, namespace, data_dir = None, file_dir = None, lock_dir = None, digest_filenames = True, **kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if file_dir is not None:
self.file_dir = file_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or file_dir is required")
else:
self.file_dir = data_dir + "/container_file"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_file_lock"
util.verify_directory(self.file_dir)
util.verify_directory(self.lock_dir)
self.lock = Synchronizer(identifier=self.namespace, use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root=self.file_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.cache')
self.hash = {}
self.debug("data file %s" % self.file)
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
**params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
if hasattr(self, 'lock_dir'):
verify_directory(self.lock_dir)
conn_params = {}
parts = url.split('?', 1)
url = parts[0]
if len(parts) > 1:
conn_params = dict(p.split('=', 1) for p in parts[1].split('&'))
netloc = urlparse(url).netloc
parts = netloc.split(':')
port = parts.pop()
if port:
port = int(port)
host = ':'.join(parts)
self.open_connection(host, port, **conn_params)
def __init__(self, namespace, url, memcache_module='auto', data_dir=None,
lock_dir=None, **kw):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mcd_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
if bmemcached is None:
raise ImportError('`bmemcached` is required.')
auth_args = []
username = kw.get('username', None)
password = kw.get('password', None)
if username:
auth_args.append(username)
if password:
auth_args.append(password)
logging.warn('WFT')
logging.warn((bmemcached, auth_args))
self.mc = MemcachedNamespaceManager.clients.get((memcache_module, url),
bmemcached.Client, url.split(';'), *auth_args)
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
if hasattr(self, 'lock_dir'):
verify_directory(self.lock_dir)
conn_params = {}
parts = url.split('?', 1)
url = parts[0]
if len(parts) > 1:
conn_params = dict(p.split('=', 1) for p in parts[1].split('&'))
netloc = urlparse(url).netloc
parts = netloc.split(':')
port = parts.pop()
if port:
port = int(port)
host = ':'.join(parts)
self.open_connection(host, port, **conn_params)
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
**params):
NamespaceManager.__init__(self, namespace)
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mongodb_gridfs_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
if not url:
raise MissingCacheParameter("MongoDB url is required")
for k, v in parse_uri(url).iteritems():
setattr(self, "url_%s" % k, v)
if not self.url_database or not self.url_nodelist:
raise MissingCacheParameter("Invalid MongoDB url.")
data_key = "mongodb_gridfs:%s:%s" % (self.url_database,
self.url_collection)
self.gridfs = MongoDBGridFSNamespaceManager.clients.get(
data_key, self._create_mongo_connection)
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, expire=None, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
if hasattr(self, 'lock_dir'):
verify_directory(self.lock_dir)
# Specify the serializer to use (pickle or json?)
self.serializer = params.pop('serializer', 'pickle')
self._expiretime = int(expire) if expire else None
conn_params = {}
parts = url.split('?', 1)
url = parts[0]
if len(parts) > 1:
conn_params = dict(p.split('=', 1) for p in parts[1].split('&'))
host, port = url.split(':', 1)
self.open_connection(host, int(port), **conn_params)
def __init__(self, namespace, url,
memcache_module='auto',
data_dir=None, lock_dir=None,
**kw):
NamespaceManager.__init__(self, namespace)
_memcache_module = _client_libs[memcache_module]
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mcd_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
# Check for pylibmc namespace manager, in which case client will be
# instantiated by subclass __init__, to handle behavior passing to the
# pylibmc client
if not _is_configured_for_pylibmc(memcache_module, _memcache_module):
self.mc = MemcachedNamespaceManager.clients.get(
(memcache_module, url),
_memcache_module.Client,
url.split(';'))
def __init__(self, namespace, bind, table, data_dir=None, lock_dir=None,
**kwargs):
"""Create a namespace manager for use with a database table via
SQLAlchemy.
``bind``
SQLAlchemy ``Engine`` or ``Connection`` object
``table``
SQLAlchemy ``Table`` object in which to store namespace data.
This should usually be something created by ``make_cache_table``.
"""
OpenResourceNamespaceManager.__init__(self, namespace)
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_db_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
self.bind = self.__class__.binds.get(str(bind.url), lambda: bind)
self.table = self.__class__.tables.get('%s:%s' % (bind.url, table.name),
lambda: table)
self.hash = {}
self._is_new = False
self.loaded = False
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
else:
self.lock_dir = None
if self.lock_dir:
verify_directory(self.lock_dir)
conn_params = {}
parts = url.split('?', 1)
url = parts[0]
if len(parts) > 1:
conn_params = dict(p.split('=', 1) for p in parts[1].split('&'))
host, port = url.split(':', 1)
self.open_connection(host, int(port), **conn_params)
def __init__(self,
namespace,
url,
data_dir=None,
lock_dir=None,
hkey='sessions',
**params):
super().__init__(namespace)
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
else:
self.lock_dir = None
if self.lock_dir:
verify_directory(self.lock_dir)
self.hkey = hkey
host, port_db = url.split(':', 1)
port, db = (int(i) for i in port_db.split('/', 1))
self.db = Redis(host=host, port=port, db=db)
log.debug(
'sessions redis backend setuped: namespace={}'.format(namespace))
def __init__(self, namespace, url=None, sa_opts=None, optimistic=False,
table_name='beaker_cache', data_dir=None, lock_dir=None,
**params):
"""Creates a database namespace manager
``url``
SQLAlchemy compliant db url
``sa_opts``
A dictionary of SQLAlchemy keyword options to initialize the engine
with.
``optimistic``
Use optimistic session locking, note that this will result in an
additional select when updating a cache value to compare version
numbers.
``table_name``
The table name to use in the database for the cache.
"""
OpenResourceNamespaceManager.__init__(self, namespace)
if sa_opts is None:
sa_opts = params
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_db_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
# Check to see if the table's been created before
url = url or sa_opts['sa.url']
table_key = url + table_name
def make_cache():
# Check to see if we have a connection pool open already
meta_key = url + table_name
def make_meta():
# SQLAlchemy pops the url, this ensures it sticks around
# later
sa_opts['sa.url'] = url
engine = sa.engine_from_config(sa_opts, 'sa.')
meta = sa.MetaData()
meta.bind = engine
return meta
meta = DatabaseNamespaceManager.metadatas.get(meta_key, make_meta)
# Create the table object and cache it now
cache = sa.Table(table_name, meta,
sa.Column('id', types.Integer, primary_key=True),
sa.Column('namespace', types.String(255), nullable=False),
sa.Column('accessed', types.DateTime, nullable=False),
sa.Column('created', types.DateTime, nullable=False),
sa.Column('data', types.PickleType, nullable=False),
sa.UniqueConstraint('namespace')
)
cache.create(checkfirst=True)
return cache
self.hash = {}
self._is_new = False
self.loaded = False
self.cache = DatabaseNamespaceManager.tables.get(table_key, make_cache)
def __init__(self, namespace, url=None, data_dir=None,
lock_dir=None, skip_pickle=False,
sparse_collection=False, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("MongoDB url is required")
if skip_pickle:
log.info("Disabling pickling for namespace: %s" % self.namespace)
self._pickle = False
if sparse_collection:
log.info(("Separating data to one row per key (sparse collection)"
" for ns %s .") % self.namespace)
self._sparse = True
# Temporarily uses a local copy of the functions until pymongo upgrades to new parser code
url_components = parse_uri(url)
host_list = url_components['nodelist']
username = url_components['username']
password = url_components['password']
collection = url_components['collection']
database = url_components['database']
options = url_components['options']
if database and host_list:
data_key = "mongodb:%s" % (database)
else:
raise MissingCacheParameter("Invalid Cache URL. Cannot parse.")
# Key will be db + collection
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mongodb_lock"
if hasattr(self, 'lock_dir'):
verify_directory(self.lock_dir)
def _create_mongo_conn():
host_uri = 'mongodb://'
for x in host_list:
host_uri += '%s:%s' % x
log.info("Host URI: %s" % host_uri)
conn = Connection(url, slave_okay=options.get('slaveok', False))
db = conn[database]
if username:
log.info("Attempting to authenticate %s/%s " % (username,
password))
if not db.authenticate(username, password):
raise InvalidCacheBackendError('Cannot authenticate to '
' MongoDB.')
return db[collection]
self.mongo = MongoDBNamespaceManager.clients.get(
data_key,
_create_mongo_conn)
def __init__(self, namespace, url=None, data_dir=None,
lock_dir=None, skip_pickle=False,
sparse_collection=False, use_file_lock=False, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("MongoDB url is required")
if skip_pickle:
log.info("Disabling pickling for namespace: %s" % self.namespace)
self._pickle = False
if sparse_collection:
log.info("Separating data to one row per key (sparse collection) for ns %s ." % self.namespace)
self._sparse = True
if use_file_lock:
log.info("Enabling file_locks for namespace: %s" % self.namespace)
self._use_file_lock = True
# Temporarily uses a local copy of the functions until pymongo upgrades to new parser code
(host_list, database, username, password, collection, options) = _parse_uri(url)
if database and host_list:
data_key = "mongodb:%s" % (database)
else:
raise MissingCacheParameter("Invalid Cache URL. Cannot parse.")
# Key will be db + collection
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mongodb_lock"
elif self._use_file_lock:
raise ImproperlyConfigured("Neither data_dir nor lock_dir are specified, while use_file_lock is set to True")
if self._use_file_lock and self.lock_dir:
verify_directory(self.lock_dir)
def _create_mongo_conn():
host_uri = 'mongodb://'
for x in host_list:
host_uri += '%s:%s' % x
log.info("Host URI: %s" % host_uri)
conn = Connection(host_uri, slave_okay=options.get('slaveok', False))
db = conn[database]
if username:
log.info("Attempting to authenticate %s/%s " % (username, password))
if not db.authenticate(username, password):
raise InvalidCacheBackendError('Cannot authenticate to '
' MongoDB.')
return db[collection]
self.mongo = MongoDBNamespaceManager.clients.get(data_key,
_create_mongo_conn)
def __init__(self, namespace, url, sa_opts=None, optimistic=False,
table_name='beaker_cache', data_dir=None, lock_dir=None,
**params):
"""Creates a database namespace manager
``url``
A SQLAlchemy database URL
``sa_opts``
A dictionary of SQLAlchemy keyword options to initialize the engine
with.
``optimistic``
Use optimistic session locking, note that this will result in an
additional select when updating a cache value to compare version
numbers.
``table_name``
The table name to use in the database for the cache.
"""
NamespaceManager.__init__(self, namespace, **params)
if sa_opts is None:
sa_opts = {}
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_db_lock"
verify_directory(self.lock_dir)
# Check to see if the table's been created before
table_key = url + str(sa_opts) + table_name
def make_cache():
# Check to see if we have a connection pool open already
meta_key = url + str(sa_opts)
def make_meta():
if url.startswith('mysql') and not sa_opts:
sa_opts['poolclass'] = pool.QueuePool
engine = sa.create_engine(url, **sa_opts)
meta = sa.BoundMetaData(engine)
return meta
meta = DatabaseNamespaceManager.metadatas.get(meta_key, make_meta)
# Create the table object and cache it now
cache = sa.Table(table_name, meta,
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('namespace', sa.String(255), nullable=False),
sa.Column('key', sa.String(255), nullable=False),
sa.Column('value', sa.BLOB(), nullable=False),
sa.UniqueConstraint('namespace', 'key')
)
cache.create(checkfirst=True)
return cache
self.cache = DatabaseNamespaceManager.tables.get(table_key, make_cache)
def __init__(self, namespace, url=None, data_dir=None,
lock_dir=None, skip_pickle=False,
sparse_collection=False, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("MongoDB url is required")
if skip_pickle:
log.info("Disabling pickling for namespace: %s" % self.namespace)
_pickle = False
if sparse_collection:
log.info("Separating data to one row per key (sparse collection) for ns %s ." % self.namespace)
self._sparse = True
conn_params = parse_mongo_url(url)
if conn_params['database'] and conn_params['host'] and \
conn_params['collection']:
data_key = "mongodb:%s#%s" % (conn_params['database'],
conn_params['collection'])
else:
raise MissingCacheParameter("Invalid Cache URL. Cannot parse"
" host, database and/or "
" collection name.")
conn_params['slave_okay'] = params.get('slave_okay') == 'True'
# Key will be db + collection
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mongodb_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
def _create_mongo_conn():
conn = pymongo.connection.Connection(conn_params['host'],\
conn_params['port'], slave_okay=conn_params['slave_okay'])
db = conn[conn_params['database']]
if conn_params['username'] and conn_params['password']:
log.info("Attempting to authenticate %s/%s " %
(conn_params['username'], conn_params['password']))
if not db.authenticate(conn_params['username'],
conn_params['password']):
raise InvalidCacheBackendError('Cannot authenticate to '
' MongoDB.')
return db[conn_params['collection']]
self.mongo = MongoDBNamespaceManager.clients.get(data_key,
_create_mongo_conn)
def __init__(self, namespace, url, data_dir=None, lock_dir=None, **params):
NamespaceManager.__init__(self, namespace, **params)
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_mcd_lock"
verify_directory(self.lock_dir)
self.mc = MemcachedNamespaceManager.clients.get(url, lambda: memcache.Client(url.split(';'), debug=0))
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mcd_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
self.mc = MemcachedNamespaceManager.clients.get(url, memcache.Client, url.split(';'))
def __init__(self, namespace, url, data_dir=None, lock_dir=None, **params):
NamespaceManager.__init__(self, namespace, **params)
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_mcd_lock"
verify_directory(self.lock_dir)
self.mc = MemcachedNamespaceManager.clients.get(
url, lambda: memcache.Client(url.split(';'), debug=0))
def __init__(self, namespace, url=None, data_dir=None,
lock_dir=None, skip_pickle=False,
sparse_collection=False, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("MongoDB url is required")
if skip_pickle:
log.info("Disabling pickling for namespace: %s" % self.namespace)
self._pickle = False
if sparse_collection:
log.info("Separating data to one row per key (sparse collection) for ns %s ." % self.namespace)
self._sparse = True
# Temporarily uses a local copy of the functions until pymongo upgrades to new parser code
(host_list, database, username, password, collection, options) = _parse_uri(url)
if database and host_list:
data_key = "mongodb:%s" % (database)
else:
raise MissingCacheParameter("Invalid Cache URL. Cannot parse.")
# Key will be db + collection
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mongodb_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
def _create_mongo_conn():
host_uri = 'mongodb://'
for x in host_list:
host_uri += '%s:%s' % x
log.info("Host URI: %s" % host_uri)
conn = Connection(host_uri, slave_okay=options.get('slaveok', False))
db = conn[database]
if username:
log.info("Attempting to authenticate %s/%s " % (username, password))
if not db.authenticate(username, password):
raise InvalidCacheBackendError('Cannot authenticate to '
' MongoDB.')
return db[collection]
self.mongo = MongoDBNamespaceManager.clients.get(data_key,
_create_mongo_conn)
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_tcd_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
host, port = url.split(':')
self.open_connection(host, int(port))
def __init__(self, namespace, url, memcache_module="auto", data_dir=None, lock_dir=None, **kw):
NamespaceManager.__init__(self, namespace)
_memcache_module = _client_libs[memcache_module]
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mcd_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
self.mc = MemcachedNamespaceManager.clients.get((memcache_module, url), _memcache_module.Client, url.split(";"))
def __init__(self, namespace, url, memcache_module='auto', data_dir=None,
lock_dir=None, **kw):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mcd_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
if ultramemcache is None:
raise ImportError('`python-ultramemcached` is required.')
_memcache_module = ultramemcache
self.mc = MemcachedNamespaceManager.clients.get((memcache_module, url),
_memcache_module.Client, url.split(';'))
def __init__(self,
namespace,
dbmmodule=None,
data_dir=None,
dbm_dir=None,
lock_dir=None,
digest_filenames=True,
**kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if dbm_dir is not None:
self.dbm_dir = dbm_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or dbm_dir is required")
else:
self.dbm_dir = data_dir + "/container_dbm"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_dbm_lock"
if dbmmodule is None:
import anydbm
self.dbmmodule = anydbm
else:
self.dbmmodule = dbmmodule
util.verify_directory(self.dbm_dir)
util.verify_directory(self.lock_dir)
self.dbm = None
self.lock = Synchronizer(identifier=self.namespace,
use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root=self.dbm_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.dbm')
self.debug("data file %s" % self.file)
self._checkfile()
def __init__(self, namespace, url, data_dir=None, lock_dir=None, **kwargs):
"""
Initialize Redis connection.
"""
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter('URL setting for Redis is required.')
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = os.path.join(data_dir, 'container_redis_lock')
if self.lock_dir:
verify_directory(self.lock_dir)
self.redis = StrictRedis.from_url(url)
def __init__(self, namespace, url=None, sa_opts=None, optimistic=False,
table_name='beaker_cache', data_dir=None, lock_dir=None,
**params):
print namespace
NamespaceManager.__init__(self, namespace)
if sa_opts is None:
sa_opts = params
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_db_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
# Check to see if the table's been created before
url = url or sa_opts['sa.url']
table_key = url + table_name
def make_table():
# Check to see if we have a connection pool open already
meta_key = url + table_name
def make_meta():
# SQLAlchemy pops the url, this ensures it sticks around
# later
sa_opts['sa.url'] = url
engine = sa.engine_from_config(sa_opts, 'sa.')
meta = sa.MetaData()
meta.bind = engine
return meta
meta = DBTableNamespaceManager.metadatas.get(meta_key, make_meta)
# Create the table object and cache it now
table = sa.Table(table_name, meta,
sa.Column('key', types.String(100), primary_key=True),
sa.Column('data', types.String(255), nullable=False),
extend_existing=True
)
table.create(checkfirst=True)
return table
self.hash = {}
self._is_new = False
self.loaded = False
self.table = DBTableNamespaceManager.tables.get(table_key, make_table)
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
**params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mcd_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
self.mc = MemcachedNamespaceManager.clients.get(
url, memcache.Client, url.split(';'))
def __init__(self, namespace, uri=None, data_dir=None, lock_dir=None, **params):
NamespaceManager.__init__(self, namespace)
if not uri:
raise MissingCacheParameter("URI is required")
self.db_connection_params = pymongo.uri_parser.parse_uri(uri)
if not self.db_connection_params["collection"]:
raise MissingCacheParameter("invalid URI: missing collection")
elif not self.db_connection_params["database"]:
raise MissingCacheParameter("invalid URI: missing database")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mongodb_lock"
if hasattr(self, "lock_dir"):
verify_directory(self.lock_dir)
self.open_connection(uri)
def __init__(self,
namespace,
url,
memcache_module='auto',
data_dir=None,
lock_dir=None,
**kw):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mcd_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
if ultramemcache is None:
raise ImportError('`python-ultramemcached` is required.')
_memcache_module = ultramemcache
self.mc = MemcachedNamespaceManager.clients.get(
(memcache_module, url), _memcache_module.Client, url.split(';'))
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
NamespaceManager.__init__(self, namespace)
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mongodb_gridfs_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
if not url:
raise MissingCacheParameter("MongoDB url is required")
for k, v in parse_uri(url).iteritems():
setattr(self, "url_%s"%k, v)
if not self.url_database or not self.url_nodelist:
raise MissingCacheParameter("Invalid MongoDB url.")
data_key = "mongodb_gridfs:%s:%s" % (self.url_database, self.url_collection)
self.gridfs = MongoDBGridFSNamespaceManager.clients.get(
data_key, self._create_mongo_connection)
def __init__(self, namespace, dbmmodule = None, data_dir = None, dbm_dir = None, lock_dir = None, digest_filenames = True, **kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if dbm_dir is not None:
self.dbm_dir = dbm_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or dbm_dir is required")
else:
self.dbm_dir = data_dir + "/container_dbm"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_dbm_lock"
if dbmmodule is None:
import anydbm
self.dbmmodule = anydbm
else:
self.dbmmodule = dbmmodule
util.verify_directory(self.dbm_dir)
util.verify_directory(self.lock_dir)
self.dbm = None
self.lock = Synchronizer(identifier=self.namespace, use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root= self.dbm_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.dbm')
self.debug("data file %s" % self.file)
self._checkfile()
def __init__(self, namespace, url=None, data_dir=None,
lock_dir=None, skip_pickle=False,
sparse_collection=False, **params):
NamespaceManager.__init__(self, namespace)
if not url:
raise MissingCacheParameter("MongoDB url is required")
if skip_pickle:
log.info("Disabling pickling for namespace: %s" % self.namespace)
self._pickle = False
if sparse_collection:
log.info("Separating data to one row per key (sparse collection) for ns %s ." % self.namespace)
self._sparse = True
# Temporarily uses a local copy of the functions until pymongo upgrades to new parser code
(host_list, database, username, password, collection, options) = _parse_uri(url)
if database and host_list:
data_key = "mongodb:%s" % (database)
else:
raise MissingCacheParameter("Invalid Cache URL. Cannot parse.")
# Key will be db + collection
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mongodb_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
def _create_mongo_conn():
store = MongoStore.get_default()
return MongoCollection(store.get_collection(collection))
self.mongo = MongoDBNamespaceManager.clients.get(data_key,
_create_mongo_conn)
def __init__(self, *args, **params):
unittest.TestCase.__init__(self, *args, **params)
# make ourselves a Myghty environment
self.root = os.path.abspath(os.path.join(os.getcwd(), 'testroot'))
# some templates
self.htdocs = os.path.join(self.root, 'htdocs')
# some more templates
self.components = os.path.join(self.root, 'components')
# data dir for cache, sessions, compiled
self.cache = os.path.join(self.root, 'cache')
# lib dir for some module components
self.lib = os.path.join(self.root, 'lib')
sys.path.insert(0, self.lib)
for path in (self.htdocs, self.components, self.cache, self.lib):
util.verify_directory(path)
self.class_set_up()
def __init__(self,
namespace,
url,
memcache_module='auto',
data_dir=None,
lock_dir=None,
**kw):
NamespaceManager.__init__(self, namespace)
_memcache_module = _client_libs[memcache_module]
if not url:
raise MissingCacheParameter("url is required")
if lock_dir:
self.lock_dir = lock_dir
elif data_dir:
self.lock_dir = data_dir + "/container_mcd_lock"
if self.lock_dir:
verify_directory(self.lock_dir)
self.mc = MemcachedNamespaceManager.clients.get(
(memcache_module, url), _memcache_module.Client, url.split(';'))
def __init__(self,
namespace,
data_dir=None,
file_dir=None,
lock_dir=None,
digest_filenames=True,
**kwargs):
NamespaceManager.__init__(self, namespace, **kwargs)
if file_dir is not None:
self.file_dir = file_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or file_dir is required")
else:
self.file_dir = data_dir + "/container_file"
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_file_lock"
util.verify_directory(self.file_dir)
util.verify_directory(self.lock_dir)
self.lock = Synchronizer(identifier=self.namespace,
use_files=True,
lock_dir=self.lock_dir,
digest_filenames=digest_filenames)
self.file = util.encoded_path(root=self.file_dir,
identifiers=[self.namespace],
digest=digest_filenames,
extension='.cache')
self.hash = {}
self.debug("data file %s" % self.file)
def __init__(self,
namespace,
url,
sa_opts=None,
optimistic=False,
table_name='beaker_cache',
data_dir=None,
lock_dir=None,
**params):
"""Creates a database namespace manager
``url``
A SQLAlchemy database URL
``sa_opts``
A dictionary of SQLAlchemy keyword options to initialize the engine
with.
``optimistic``
Use optimistic session locking, note that this will result in an
additional select when updating a cache value to compare version
numbers.
``table_name``
The table name to use in the database for the cache.
"""
NamespaceManager.__init__(self, namespace, **params)
if sa_opts is None:
sa_opts = {}
if lock_dir is not None:
self.lock_dir = lock_dir
elif data_dir is None:
raise MissingCacheParameter("data_dir or lock_dir is required")
else:
self.lock_dir = data_dir + "/container_db_lock"
verify_directory(self.lock_dir)
# Check to see if the table's been created before
table_key = url + str(sa_opts) + table_name
def make_cache():
# Check to see if we have a connection pool open already
meta_key = url + str(sa_opts)
def make_meta():
if url.startswith('mysql') and not sa_opts:
sa_opts['poolclass'] = pool.QueuePool
engine = sa.create_engine(url, **sa_opts)
meta = sa.BoundMetaData(engine)
return meta
meta = DatabaseNamespaceManager.metadatas.get(meta_key, make_meta)
# Create the table object and cache it now
cache = sa.Table(
table_name, meta, sa.Column('id', sa.Integer,
primary_key=True),
sa.Column('namespace', sa.String(255), nullable=False),
sa.Column('key', sa.String(255), nullable=False),
sa.Column('value', sa.BLOB(), nullable=False),
sa.UniqueConstraint('namespace', 'key'))
cache.create(checkfirst=True)
return cache
self.cache = DatabaseNamespaceManager.tables.get(table_key, make_cache)
def _ensuredir(self, filename):
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
util.verify_directory(dirname)
def _ensuredir(self):
if not os.path.exists(self.lock_dir):
util.verify_directory(self.lock_dir)
def _ensuredir(self):
if not os.path.exists(self.lock_dir):
util.verify_directory(self.lock_dir)
def create_directory(self, dir, path):
util.verify_directory(os.path.join(dir, path))
