def test_request_aware_read_current(foo): # noqa
request1 = make_mocked_request('POST', '/')
foo._p_jar.transaction_manager.begin(request1)
request2 = make_mocked_request('POST', '/')
foo._p_jar.transaction_manager.begin(request2)
for i in range(0, 5):
SetItemView(foo, request1)(str(i), OOBTree.OOBTree())
assert '_txn_readCurrent' in request1.__dict__
assert len(request1._txn_readCurrent) > 0
CommitView(foo, request1)()
assert len(request1._txn_readCurrent) == 0
for i in range(5, 10):
SetItemView(foo, request2)(str(i), OOBTree.OOBTree())
assert '_txn_readCurrent' in request2.__dict__
assert len(request2._txn_readCurrent) > 0
CommitView(foo, request2)()
assert len(request2._txn_readCurrent) == 0
def test_concurrent_transaction_abort_has_no_side_effects(root, foo, bar): # noqa
tm = root._p_jar.transaction_manager
# Create /foo/a
request1 = make_mocked_request('POST', '/foo')
t1 = tm.begin(request1)
SetItemView(foo, request1)('a', OOBTree.OOBTree())
# Test that object is registered
assert foo in request1._txn_dm._registered_objects
# Create /bar/b
request2 = make_mocked_request('POST', '/bar')
t2 = tm.begin(request2)
SetItemView(bar, request2)('b', OOBTree.OOBTree())
# Test that object is registered
assert bar in request2._txn_dm._registered_objects
# Test that two transactions are indepedent
assert bar not in request1._txn_dm._registered_objects
assert foo not in request2._txn_dm._registered_objects
# Abort the first transaction
t1.abort()
# Commit the second transaction
t2.commit()
# Test that /foo/a has not been created by t1
assert 'a' not in foo
# Test that /bar/b has been created by t2
assert 'b' in bar
def __init__(self, *args, **kw):
super(TrackingStorage, self).__init__(*args, **kw)
self.indexes = OOBTree.OOBTree()
for idx in self.indexAttributes:
self.indexes[idx] = FieldIndex()
self.runs = IOBTree.IOBTree()
self.currentRuns = OOBTree.OOBTree()
self.taskUsers = OOBTree.OOBTree()
def _reset(self):
# Map tagid to tag object
self._tagid_to_obj = IOBTree.IOBTree()
# Indexes
self._user_to_tagids = OOBTree.OOBTree()
self._item_to_tagids = IOBTree.IOBTree()
self._name_to_tagids = OOBTree.OOBTree()
self._community_to_tagids = OOBTree.OOBTree()
def test_unpack_nested_btree_treeset(self):
struct = OOBTree.OOBTree()
struct['foo'] = OOBTree.OOBTree()
struct['foo']['fooz'] = OOBTree.OOTreeSet()
struct['foo']['fooz'].insert('baz')
struct['foo']['fooz'].insert('bar')
g = NetworkGraph()
flat = g.unpack(struct)
self.assertEqual(flat, {'foo': {'fooz': ['bar', 'baz']}})
def __init__(self):
self._v_current_card = 0
self._v_text = None
self._v_cardNumber = None
self.last_id = 0
self.card_list = PersistentList()
self.to_be_indexed = PersistentList()
self.card_store = OOBTree.OOBTree()
self.hint_store = OOBTree.OOBTree()
self.CreateCard(at_end=1, and_set=0)
def __init__(self, context=None):
self._mtime = 0
# primary storage: (long statusid) -> (object IStatusUpdate)
self._status_mapping = LOBTree.LOBTree()
# index by user: (string userid) -> (object TreeSet(long statusid))
self._user_mapping = OOBTree.OOBTree()
# index by tag: (string tag) -> (object TreeSet(long statusid))
self._tag_mapping = OOBTree.OOBTree()
# index by context (string UUID) -> (object TreeSet(long statusid))
self._uuid_mapping = OOBTree.OOBTree()
def _reset(self):
# Map tagid to tag object
self._tagid_to_obj = IOBTree.IOBTree()
# Indexes
self._user_to_tagids = OOBTree.OOBTree()
self._item_to_tagids = IOBTree.IOBTree()
self._name_to_tagids = OOBTree.OOBTree()
self._community_to_tagids = OOBTree.OOBTree()
# Precomputed cloud data
self._global_cloud = PersistentMapping()
self._community_clouds = OOBTree.OOBTree()
def test_transaction_proxy(root, foo, bar):
tm = root._p_jar.transaction_manager
# Create /foo/a
request1 = make_mocked_request('POST', '/foo')
t1 = tm.begin(request1)
SetItemView(foo, request1)('a', OOBTree.OOBTree())
# Test that object is registered
assert hasattr(request1, '_txn')
assert hasattr(request1, '_txn_time')
assert hasattr(request1, '_txn_dm')
assert foo in request1._txn_dm._registered_objects
# Create /bar/b
request2 = TransactionProxy(request1)
t2 = tm.begin(request2)
SetItemView(bar, request2)('b', OOBTree.OOBTree())
# Test that object is registered
assert hasattr(request2, '_txn')
assert hasattr(request2, '_txn_time')
assert hasattr(request2, '_txn_dm')
assert bar in request2._txn_dm._registered_objects
# Test that two transactions are indepedent
assert t1 is not t2
assert t1 is request1._txn
assert t2 is request2._txn
assert request1._txn_time < request2._txn_time
assert bar not in request1._txn_dm._registered_objects
assert foo not in request2._txn_dm._registered_objects
assert len(request1._txn_dm._registered_objects) == 1
assert len(request2._txn_dm._registered_objects) == 1
# Create /bar/c
SetItemView(bar, request2)('c', OOBTree.OOBTree())
# Test that registered objects are not affected
assert len(request1._txn_dm._registered_objects) == 1
assert len(request2._txn_dm._registered_objects) == 1
# Commit
t1.commit()
t2.commit()
# Test that /foo/a and /bar/b have different transaction
assert foo['a']._p_serial != bar['b']._p_serial
# Test that /bar/b and /bar/c have the same transaction
assert bar['b']._p_serial == bar['c']._p_serial
def reinit(self):
"""Remove old DB and rebuild it from scratch."""
self._root = None
transaction.abort()
self._connection.close()
self._db = None
for f in glob.glob(p.join(self.cache_dir, "Data.fs*")):
os.unlink(f)
self._db = ZODB.DB(
ZODB.FileStorage.FileStorage(p.join(self.cache_dir, 'Data.fs')))
self._connection = self._db.open()
self._root = self._connection.root()
self._root['advisory'] = OOBTree.OOBTree()
self._root['by_product'] = OOBTree.OOBTree()
self._root['meta'] = Meta()
def create_zodb(zodb_URI):
''' zodb_URI example:
zeo://localhost:1234
'''
db = connect_zodb(zodb_URI)
if not db.has_key('simulations'):
db['simulations'] = OOBTree.OOBTree()
if not db.has_key('sim_groups'):
db['sim_groups'] = OOBTree.OOBTree()
transaction.commit()
# setup predefined groups
groups = db['sim_groups']
setup_group(db, 'TEST', 'lyx', None, 'Simulation group for testing')
setup_group(db, 'STANDARD', 'lyx', None, 'A standard simulation group')
def __init__(self, context=None):
# last write stamp in milliseconds
self._mtime = 0
# primary storage: (long statusid) -> (object IStatusUpdate)
self._status_mapping = LOBTree.LOBTree()
# index by user: (string userid) -> (object TreeSet(long statusid))
self._user_mapping = OOBTree.OOBTree()
# index by tag: (string tag) -> (object TreeSet(long statusid))
self._tag_mapping = OOBTree.OOBTree()
# index by context (string UUID) -> (object TreeSet(long statusid))
self._uuid_mapping = OOBTree.OOBTree()
# index by thread (string UUID) -> (object TreeSet(long statusid))
self._threadid_mapping = OOBTree.OOBTree()
# index by mentions (string UUID) -> (object TreeSet(long statusid))
self._mentions_mapping = OOBTree.OOBTree()
def setup_uuid_mapping(context):
"""0.5 adds a new index"""
tool = queryUtility(IMicroblogTool)
if not hasattr(tool, '_uuid_mapping'):
logger.info("Adding missing UUID mapping to %s" % repr(tool))
tool._uuid_mapping = OOBTree.OOBTree()
commit()
def test_multiple_connections(conn1, conn2): # noqa
root1 = conn1.root()
root2 = conn2.root()
request = make_mocked_request('POST', '/')
txn = root1._p_jar.transaction_manager.begin(request)
SetItemView(root1, request)('foo', OOBTree.OOBTree())
txn.commit()
assert 'foo' in root1
assert 'foo' not in root2
conn2.newTransaction(None)
assert 'foo' in root2
request1 = make_mocked_request('POST', '/')
txn1 = root1._p_jar.transaction_manager.begin(request1)
SetItemView(root1['foo'], request1)('bar', True)
request2 = make_mocked_request('POST', '/')
txn2 = root2._p_jar.transaction_manager.begin(request2)
SetItemView(root2['foo'], request2)('bar', False)
txn1.commit()
assert root1['foo']['bar']
assert not root2['foo']['bar']
with pytest.raises(ConflictError):
txn2.commit()
assert root2['foo']['bar']
def __enter__(self):
storage = ZODB.FileStorage.FileStorage(self.cache_dir + '/' +
'Data.fs')
self._db = ZODB.DB(storage)
self._connection = self._db.open()
self._root = self._connection.root()
self._root.setdefault('meta', Meta())
self._root.setdefault('archives', OOBTree.OOBTree())
def getSessionManager(debug=0):
root = DBMgr.getInstance().getDBConnection().root()
try:
sm = root["SessionManagers"]["main"]
except KeyError, e:
sm = PSessionManager()
root["SessionManagers"] = OOBTree.OOBTree()
root["SessionManagers"]["main"] = sm
def __init__(self, host):
self.host = ftputil.FTPHost(host,
settings.username,
settings.password,
session_factory=ftplib.FTP)
self.product = "ftp"
# required
self.root = OOBTree.OOBTree()
def __init__(self, name):
"""Initialize new chat session.
name -- the channel's name
"""
self.name = name
# Internal attribute: _messages holds all the chat messages.
self._messages = OOBTree.OOBTree()
def getMaKaCInfoInstance(cls):
dbmgr = DBMgr.getInstance()
root = dbmgr.getDBConnection().root()
try:
minfo = root["MaKaCInfo"]["main"]
except KeyError, e:
minfo = MaKaCInfo()
root["MaKaCInfo"] = OOBTree.OOBTree()
root["MaKaCInfo"]["main"] = minfo
def getTaskListInstance(cls):
dbmgr = db.DBMgr.getInstance()
root = dbmgr.getDBConnection().root()
try:
tlist = root["taskList"]["main"]
except KeyError, e:
tlist = taskList()
root["taskList"] = OOBTree.OOBTree()
root["taskList"]["main"] = tlist
def evolve(site):
"""
Add precomputed cloud data to Tags.
"""
tags = find_tags(site)
tags._global_cloud = PersistentMapping()
tags._community_clouds = OOBTree.OOBTree()
for tag in tags._tagid_to_obj.values():
update_clouds(tags, tag)
def _getConfRegistry( self ):
if not self._confRegistry:
db_root = DBMgr.getInstance().getDBConnection().root()
if db_root.has_key( "webfactoryregistry" ):
self._confRegistry = db_root["webfactoryregistry"]
else:
self._confRegistry = OOBTree.OOBTree()
db_root["webfactoryregistry"] = self._confRegistry
return self._confRegistry
def test_concurrent_btree_write(foo, conn2): # noqa
bar = conn2.root()['foo']
# Init a few buckets
request = make_mocked_request('POST', '/')
foo._p_jar.transaction_manager.begin(request)
for i in range(0, 50):
SetItemView(foo, request)(str(i), OOBTree.OOBTree())
CommitView(foo, request)()
assert '1' in foo
assert '1' not in bar
bar._p_jar.newTransaction(None)
assert '1' in bar
# Concurrent requests on different connections
request1 = make_mocked_request('POST', '/')
foo._p_jar.transaction_manager.begin(request1)
request2 = make_mocked_request('POST', '/')
bar._p_jar.transaction_manager.begin(request2)
# Mutate one bucket
for i in range(100, 101):
SetItemView(foo, request1)(str(i), OOBTree.OOBTree())
assert '_txn_readCurrent' in request1.__dict__
assert len(request1._txn_readCurrent) > 0
CommitView(foo, request1)()
assert len(request1._txn_readCurrent) == 0
# Mutate another bucket on another connection without sync
for i in range(500, 501):
SetItemView(bar, request2)(str(i), OOBTree.OOBTree())
assert '_txn_readCurrent' in request2.__dict__
assert len(request2._txn_readCurrent) > 0
CommitView(bar, request2)()
assert len(request2._txn_readCurrent) == 0
def getMaKaCInfoInstance(cls):
dbmgr = db.DBMgr.getInstance()
root = dbmgr.getDBConnection().root()
try:
minfo = root['MaKaCInfo']['main']
except KeyError:
minfo = MaKaCInfo()
root['MaKaCInfo'] = OOBTree.OOBTree()
root['MaKaCInfo']['main'] = minfo
return minfo
def _getInternalPagesMgrRegistery( self ):
#DBMgr.getInstance().commit()
if not self._pagesMgrRegistery:
db_root = DBMgr.getInstance().getDBConnection().root()
if db_root.has_key( "internalPagesRegistery" ):
self._pagesMgrRegistery = db_root["internalPagesRegistery"]
else:
self._pagesMgrRegistery = OOBTree.OOBTree()
db_root["internalPagesRegistery"] = self._pagesMgrRegistery
return self._pagesMgrRegistery
def getRepositoryFromDB(cls):
from indico.core.db import DBMgr
dbRoot = DBMgr.getInstance().getDBConnection().root()
try:
fr = dbRoot["local_repositories"][cls._repo_name]
except KeyError:
fr = cls()
if not "local_repositories" in dbRoot:
dbRoot["local_repositories"] = OOBTree.OOBTree()
dbRoot["local_repositories"][cls._repo_name] = fr
return fr
async def connect(self, opsdroid=None):
_LOGGER.debug("Connecting ZODB")
filepath = self.config.get("database", "opsdroid.fs")
self.storage = ZODB.FileStorage.FileStorage(filepath)
self.db = ZODB.DB(self.storage)
self.connection = self.db.open()
self.root = self.connection.root
if not hasattr(self.root, "memory"):
with transaction.manager:
self.root.memory = OOBTree.OOBTree()
self.memory = self.root.memory
_LOGGER.info("ZODB database connected")
def reindex(self):
"""Regenerate product index."""
_log.info('Reindexing database')
del self._root['by_product']
bp = OOBTree.OOBTree()
for vuln in self._root['advisory'].values():
if vuln.nodes:
for prod in (n.product for n in vuln.nodes):
bp.setdefault(prod, [])
bp[prod].append(vuln)
self._root['by_product'] = bp
transaction.commit()
def __init__(self, context=None):
# last add in milliseconds - used in both async and cache key
self._mtime = 0
# cache buster in MICROseconds - used in cache key only
self._ctime = 0
# primary storage: (long statusid) -> (object IStatusUpdate)
self._status_mapping = LOBTree.LOBTree()
# archive deleted: (long statusid) -> (object IStatusUpdate)
self._status_archive = LOBTree.LOBTree()
# index by user: (string userid) -> (object TreeSet(long statusid))
self._user_mapping = OOBTree.OOBTree()
# index by tag: (string tag) -> (object TreeSet(long statusid))
self._tag_mapping = OOBTree.OOBTree()
# index by microblog_context:
# (string UUID) -> (object TreeSet(long statusid))
self._uuid_mapping = OOBTree.OOBTree() # keep old name for backcompat
# index by content_context:
# (string UUID) -> (object TreeSet(long statusid))
self._content_uuid_mapping = OOBTree.OOBTree()
# index by thread (string UUID) -> (object TreeSet(long statusid))
self._threadid_mapping = OOBTree.OOBTree()
# index by mentions (string UUID) -> (object TreeSet(long statusid))
self._mentions_mapping = OOBTree.OOBTree()
# index all content updates: (object TreeSet(long statusid))
self._is_content_mapping = LLBTree.LLTreeSet()
# index all human updates: (object TreeSet(long statusid))
self._is_human_mapping = LLBTree.LLTreeSet()
def __enter__(self):
"""Keeps database connection open while in this context."""
_log.debug('Opening database in %s', self.cache_dir)
os.makedirs(self.cache_dir, exist_ok=True)
self.lock()
self._db = ZODB.DB(
ZODB.FileStorage.FileStorage(p.join(self.cache_dir, 'Data.fs')))
self._connection = self._db.open()
self._root = self._connection.root()
try:
self._root.setdefault('advisory', OOBTree.OOBTree())
self._root.setdefault('by_product', OOBTree.OOBTree())
self._root.setdefault('meta', Meta())
# may trigger exceptions if the database is inconsistent
list(self._root['by_product'].keys())
if 'archives' in self._root:
_log.warn('Pre-1.9.0 database found - rebuilding')
self.reinit()
except (TypeError, EOFError):
_log.warn('Incompatible objects found in database - rebuilding DB')
self.reinit()
return self