class MemoryNamespaceManager(NamespaceManager): namespaces = util.SyncDict(_threading.Lock(), {}) def __init__(self, namespace, **kwargs): NamespaceManager.__init__(self, namespace, **kwargs) self.lock = Synchronizer( identifier="memorycontainer/namespacelock/%s" % self.namespace, use_files=False) self.dictionary = MemoryNamespaceManager.namespaces.get( self.namespace, lambda: {}) def do_acquire_read_lock(self): self.lock.acquire_read_lock() def do_release_read_lock(self): self.lock.release_read_lock() def do_acquire_write_lock(self, wait=True): return self.lock.acquire_write_lock(wait) def do_release_write_lock(self): self.lock.release_write_lock() # the open and close methods are totally overridden to eliminate # the unnecessary "open count" computation involved def open(self, *args, **kwargs): pass def close(self, *args, **kwargs): pass def __getitem__(self, key): return self.dictionary[key] def __contains__(self, key): return self.dictionary.__contains__(key) def has_key(self, key): return self.dictionary.__contains__(key) def __setitem__(self, key, value): self.dictionary[key] = value def __delitem__(self, key): del self.dictionary[key] def do_remove(self): self.dictionary.clear() def keys(self): return self.dictionary.keys()
def __init__(self, namespace, **kwargs): self.namespace = namespace self.openers = 0 self.mutex = _threading.Lock()
def __init__(self, namespace): NamespaceManager.__init__(self, namespace) self.access_lock = self.get_access_lock() self.openers = 0 self.mutex = _threading.Lock()
def __init__(self, clsmap): self._clsmap = clsmap self._mutex = _threading.Lock()
class DatabaseNamespaceManager(NamespaceManager): metadatas = SyncDict(_threading.Lock(), {}) tables = SyncDict(_threading.Lock(), {}) def __init__(self, namespace, url, sa_opts=None, optimistic=False, table_name='beaker_cache', data_dir=None, lock_dir=None, **params): """Creates a database namespace manager ``url`` A SQLAlchemy database URL ``sa_opts`` A dictionary of SQLAlchemy keyword options to initialize the engine with. ``optimistic`` Use optimistic session locking, note that this will result in an additional select when updating a cache value to compare version numbers. ``table_name`` The table name to use in the database for the cache. """ NamespaceManager.__init__(self, namespace, **params) if sa_opts is None: sa_opts = {} if lock_dir is not None: self.lock_dir = lock_dir elif data_dir is None: raise MissingCacheParameter("data_dir or lock_dir is required") else: self.lock_dir = data_dir + "/container_db_lock" verify_directory(self.lock_dir) # Check to see if the table's been created before table_key = url + str(sa_opts) + table_name def make_cache(): # Check to see if we have a connection pool open already meta_key = url + str(sa_opts) def make_meta(): if url.startswith('mysql') and not sa_opts: sa_opts['poolclass'] = pool.QueuePool engine = sa.create_engine(url, **sa_opts) meta = sa.BoundMetaData(engine) return meta meta = DatabaseNamespaceManager.metadatas.get(meta_key, make_meta) # Create the table object and cache it now cache = sa.Table( table_name, meta, sa.Column('id', sa.Integer, primary_key=True), sa.Column('namespace', sa.String(255), nullable=False), sa.Column('key', sa.String(255), nullable=False), sa.Column('value', sa.BLOB(), nullable=False), sa.UniqueConstraint('namespace', 'key')) cache.create(checkfirst=True) return cache self.cache = DatabaseNamespaceManager.tables.get(table_key, make_cache) # The database does its own locking. override our own stuff def do_acquire_read_lock(self): pass def do_release_read_lock(self): pass def do_acquire_write_lock(self, wait=True): return True def do_release_write_lock(self): pass # override open/close to do nothing, keep the connection open as long # as possible def open(self, *args, **params): pass def close(self, *args, **params): pass def __getitem__(self, key): cache = self.cache result = sa.select([cache.c.value], sa.and_(cache.c.namespace == self.namespace, cache.c.key == key)).execute() rows = result.fetchall() if len(rows) > 0: return cPickle.loads(str(rows[0]['value'])) else: raise KeyError(key) def __contains__(self, key): cache = self.cache rows = sa.select([cache.c.id], sa.and_(cache.c.namespace == self.namespace, cache.c.key == key)).execute().fetchall() return len(rows) > 0 def has_key(self, key): cache = self.cache rows = sa.select([cache.c.id], sa.and_(cache.c.namespace == self.namespace, cache.c.key == key)).execute().fetchall() return len(rows) > 0 def __setitem__(self, key, value): cache = self.cache rows = sa.select([cache.c.id], sa.and_(cache.c.namespace == self.namespace, cache.c.key == key)).execute().fetchall() value = cPickle.dumps(value) if len(rows) > 0: id = rows[0]['id'] cache.update(cache.c.id == id).execute(value=value) else: cache.insert().execute(namespace=self.namespace, key=key, value=value) def __delitem__(self, key): cache = self.cache cache.delete( sa.and_(cache.c.namespace == self.namespace, cache.c.key == key)).execute() def do_remove(self): cache = self.cache cache.delete(cache.c.namespace == self.namespace).execute() def keys(self): cache = self.cache rows = sa.select( [cache.c.key], cache.c.namespace == self.namespace).execute().fetchall() return [x['key'] for x in rows]
class MemcachedNamespaceManager(NamespaceManager): clients = SyncDict(_threading.Lock(), {}) def __init__(self, namespace, url, data_dir=None, lock_dir=None, **params): NamespaceManager.__init__(self, namespace, **params) if lock_dir is not None: self.lock_dir = lock_dir elif data_dir is None: raise MissingCacheParameter("data_dir or lock_dir is required") else: self.lock_dir = data_dir + "/container_mcd_lock" verify_directory(self.lock_dir) self.mc = MemcachedNamespaceManager.clients.get( url, lambda: memcache.Client(url.split(';'), debug=0)) # memcached does its own locking. override our own stuff def do_acquire_read_lock(self): pass def do_release_read_lock(self): pass def do_acquire_write_lock(self, wait=True): return True def do_release_write_lock(self): pass # override open/close to do nothing, keep memcache connection open as long # as possible def open(self, *args, **params): pass def close(self, *args, **params): pass def __getitem__(self, key): cache_key = key.replace(' ', '\302\267') keys = [self.namespace + '_' + cache_key, self.namespace + ':keys'] key_dict = self.mc.get_multi(keys) if cache_key not in key_dict.get(self.namespace + ':keys', {}): raise KeyError(key) return key_dict[self.namespace + '_' + cache_key] def __contains__(self, key): return self.has_key(key) def has_key(self, key): key = key.replace(' ', '\302\267') keys = self.mc.get(self.namespace + ':keys') or {} return key in keys def __setitem__(self, key, value): key = key.replace(' ', '\302\267') keys = self.mc.get(self.namespace + ':keys') if keys is None: keys = {} keys[key] = True self.mc.set(self.namespace + ':keys', keys) self.mc.set(self.namespace + "_" + key, value) def __delitem__(self, key): key = key.replace(' ', '\302\267') keys = self.mc.get(self.namespace + ':keys') try: del keys[key] self.mc.delete(self.namespace + "_" + key) self.mc.set(self.namespace + ':keys', keys) except KeyError: raise def do_remove(self): keys = self.mc.get(self.namespace + ':keys') if keys is not None: delete_keys = [self.namespace + '_' + x for x in keys] delete_keys.append(self.namespace + ':keys') self.mc.delete_multi(delete_keys) def keys(self): keys = self.mc.get(self.namespace + ':keys') if keys is None: return [] else: return [x.replace('\302\267', ' ') for x in keys.keys()]
class DatabaseNamespaceManager(NamespaceManager): metadatas = SyncDict(_threading.Lock(), {}) tables = SyncDict(_threading.Lock(), {}) def __init__(self, namespace, url=None, sa_opts=None, optimistic=False, table_name='beaker_cache', data_dir=None, lock_dir=None, **params): """Creates a database namespace manager ``url`` SQLAlchemy compliant db url ``sa_opts`` A dictionary of SQLAlchemy keyword options to initialize the engine with. ``optimistic`` Use optimistic session locking, note that this will result in an additional select when updating a cache value to compare version numbers. ``table_name`` The table name to use in the database for the cache. """ NamespaceManager.__init__(self, namespace, **params) if sa_opts is None: sa_opts = params if lock_dir is not None: self.lock_dir = lock_dir elif data_dir is None: raise MissingCacheParameter("data_dir or lock_dir is required") else: self.lock_dir = data_dir + "/container_db_lock" verify_directory(self.lock_dir) # Check to see if the table's been created before url = url or sa_opts['sa.url'] table_key = url + table_name def make_cache(): # Check to see if we have a connection pool open already meta_key = url + table_name def make_meta(): if sa_version == '0.3': if url.startswith('mysql') and not sa_opts: sa_opts['poolclass'] = pool.QueuePool engine = sa.create_engine(url, **sa_opts) meta = sa.BoundMetaData(engine) else: # SQLAlchemy pops the url, this ensures it sticks around # later sa_opts['sa.url'] = url engine = sa.engine_from_config(sa_opts, 'sa.') meta = sa.MetaData() meta.bind = engine return meta meta = DatabaseNamespaceManager.metadatas.get(meta_key, make_meta) # Create the table object and cache it now cache = sa.Table(table_name, meta, sa.Column('id', types.Integer, primary_key=True), sa.Column('namespace', types.String(255), nullable=False), sa.Column('accessed', types.DateTime, nullable=False), sa.Column('created', types.DateTime, nullable=False), sa.Column('data', types.BLOB(), nullable=False), sa.UniqueConstraint('namespace') ) cache.create(checkfirst=True) return cache self.hash = {} self._is_new = False self.loaded = False self.cache = DatabaseNamespaceManager.tables.get(table_key, make_cache) # The database does its own locking. override our own stuff def do_acquire_read_lock(self): pass def do_release_read_lock(self): pass def do_acquire_write_lock(self, wait = True): return True def do_release_write_lock(self): pass def do_open(self, flags): # If we already loaded the data, don't bother loading it again if self.loaded: self.flags = flags return cache = self.cache result = sa.select([cache.c.data], cache.c.namespace==self.namespace ).execute().fetchone() if not result: self._is_new = True self.hash = {} else: self._is_new = False try: self.hash = cPickle.loads(str(result['data'])) except (IOError, OSError, EOFError, cPickle.PickleError): log.debug("Couln't load pickle data, creating new storage") self.hash = {} self._is_new = True self.flags = flags self.loaded = True def do_close(self): if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): cache = self.cache if self._is_new: cache.insert().execute(namespace=self.namespace, data=cPickle.dumps(self.hash), accessed=datetime.now(), created=datetime.now()) self._is_new = False else: cache.update(cache.c.namespace==self.namespace).execute( data=cPickle.dumps(self.hash), accessed=datetime.now()) self.flags = None def do_remove(self): cache = self.cache cache.delete(cache.c.namespace==self.namespace).execute() self.hash = {} # We can retain the fact that we did a load attempt, but since the # file is gone this will be a new namespace should it be saved. self._is_new = True def __getitem__(self, key): return self.hash[key] def __contains__(self, key): return self.hash.has_key(key) def __setitem__(self, key, value): self.hash[key] = value def __delitem__(self, key): del self.hash[key] def keys(self): return self.hash.keys()
class SQLAlchemyNamespaceManager(NamespaceManager): binds = SyncDict(_threading.Lock(), {}) tables = SyncDict(_threading.Lock(), {}) def __init__(self, namespace, bind, table, data_dir=None, lock_dir=None, **kwargs): """Create a namespace manager for use with a database table via SQLAlchemy. ``bind`` SQLAlchemy ``Engine`` or ``Connection`` object ``table`` SQLAlchemy ``Table`` object in which to store namespace data. This should usually be something created by ``make_cache_table``. """ NamespaceManager.__init__(self, namespace, **kwargs) if lock_dir is not None: self.lock_dir = lock_dir elif data_dir is None: raise MissingCacheParameter('data_dir or lock_dir is required') else: self.lock_dir = data_dir + '/container_db_lock' verify_directory(self.lock_dir) self.bind = self.__class__.binds.get(str(bind.url), lambda: bind) self.table = self.__class__.tables.get('%s:%s' % (bind.url, table.name), lambda: table) self.hash = {} self._is_new = False self.loaded = False def do_acquire_read_lock(self): pass def do_release_read_lock(self): pass def do_acquire_write_lock(self, wait=True): return True def do_release_write_lock(self): pass def do_open(self, flags): if self.loaded: self.flags = flags return select = sa.select([self.table.c.data], (self.table.c.namespace == self.namespace)) result = self.bind.execute(select).fetchone() if not result: self._is_new = True self.hash = {} else: self._is_new = False try: self.hash = cPickle.loads(str(result['data'])) except (IOError, OSError, EOFError, cPickle.PickleError): log.debug("Couln't load pickle data, creating new storage") self.hash = {} self._is_new = True self.flags = flags self.loaded = True def do_close(self): if self.flags is not None and (self.flags == 'c' or self.flags == 'w'): data = cPickle.dumps(self.hash) if self._is_new: insert = self.table.insert() self.bind.execute(insert, namespace=self.namespace, data=data, accessed=datetime.now(), created=datetime.now()) self._is_new = False else: update = self.table.update(self.table.c.namespace == self.namespace) self.bind.execute(update, data=data, accessed=datetime.now()) self.flags = None def do_remove(self): delete = self.table.delete(self.table.c.namespace == self.namespace) self.bind.execute(delete) self.hash = {} self._is_new = True def __getitem__(self, key): return self.hash[key] def __contains__(self, key): return self.hash.has_key(key) def __setitem__(self, key, value): self.hash[key] = value def __delitem__(self, key): del self.hash[key] def keys(self): return self.hash.keys()