def check_record_iternext(self):
from ZODB.DB import DB
db = DB(self._storage)
conn = db.open()
conn.root()['abc'] = MinPO('abc')
conn.root()['xyz'] = MinPO('xyz')
transaction.commit()
# Ensure it's all on disk.
db.close()
self._storage.close()
self.open()
key = None
for x in ('\000', '\001', '\002'):
oid, tid, data, next_oid = self._storage.record_iternext(key)
self.assertEqual(oid, ('\000' * 7) + x)
key = next_oid
expected_data, expected_tid = self._storage.load(oid, '')
self.assertEqual(expected_data, data)
self.assertEqual(expected_tid, tid)
if x == '\002':
self.assertEqual(next_oid, None)
else:
self.assertNotEqual(next_oid, None)
def checkPackBatchLockNoWait(self):
# Exercise the code in the pack algorithm that attempts to get the
# commit lock but will sleep if the lock is busy.
self._storage = self.make_storage(pack_batch_timeout=0)
adapter = self._storage._adapter
test_conn, test_cursor = adapter.connmanager.open()
slept = []
def sim_sleep(seconds):
slept.append(seconds)
adapter.locker.release_commit_lock(test_cursor)
test_conn.rollback()
adapter.connmanager.close(test_conn, test_cursor)
db = DB(self._storage)
try:
# add some data to be packed
c = db.open()
r = c.root()
r['alpha'] = PersistentMapping()
transaction.commit()
del r['alpha']
transaction.commit()
# Pack, with a commit lock held
now = packtime = time.time()
while packtime <= now:
packtime = time.time()
adapter.locker.hold_commit_lock(test_cursor)
self._storage.pack(packtime, referencesf, sleep=sim_sleep)
self.assertTrue(len(slept) > 0)
finally:
db.close()
def checkPackWhileReferringObjectChanges(self):
# Packing should not remove objects referenced by an
# object that changes during packing.
db = DB(self._storage)
try:
# add some data to be packed
c = db.open()
root = c.root()
child = PersistentMapping()
root['child'] = child
transaction.commit()
expect_oids = [child._p_oid]
def inject_changes():
# Change the database just after the list of objects
# to analyze has been determined.
child2 = PersistentMapping()
root['child2'] = child2
transaction.commit()
expect_oids.append(child2._p_oid)
adapter = self._storage._adapter
adapter.packundo.on_filling_object_refs = inject_changes
packtime = time.time()
self._storage.pack(packtime, referencesf)
# "The on_filling_object_refs hook should have been called once")
self.assertEqual(len(expect_oids), 2, expect_oids)
# Both children should still exist.
self._storage.load(expect_oids[0], '')
self._storage.load(expect_oids[1], '')
finally:
db.close()
def test_storage_has_data(self):
from relstorage.zodbconvert import storage_has_data
src = FileStorage(self.srcfile, create=True)
self.assertFalse(storage_has_data(src))
db = DB(src) # add the root object
db.close()
self.assertTrue(storage_has_data(src))
def zodb_root(zodb_storage):
"""Return root object of opened ZODB storage."""
transaction.abort()
db = DB(zodb_storage)
connection = db.open()
yield connection.root()
connection.close()
def get_db_connection(blob_dir):
storage = MappingStorage('test')
blob_storage = BlobStorage(blob_dir, storage)
db = DB(blob_storage)
conn = db.open()
create_app_root(conn)
return conn
def checkNonASCIITransactionMetadata(self):
# Verify the database stores and retrieves non-ASCII text
# in transaction metadata.
ugly_string = ''.join(chr(c) for c in range(256))
if not isinstance(ugly_string, bytes):
# Py3
check_string = ugly_string.encode("latin-1")
else:
check_string = ugly_string
db = DB(self._storage)
try:
c1 = db.open()
r1 = c1.root()
r1['alpha'] = 1
transaction.get().setUser(ugly_string)
transaction.commit()
r1['alpha'] = 2
transaction.get().note(ugly_string)
transaction.commit()
info = self._storage.undoInfo()
self.assertEqual(info[0]['description'], check_string)
self.assertEqual(info[1]['user_name'], b'/ ' + check_string)
finally:
db.close()
def testDeepCopyCanInvalidate(self):
"""
Tests regression for invalidation problems related to missing
readers and writers values in cloned objects (see
http://mail.zope.org/pipermail/zodb-dev/2008-August/012054.html)
"""
import ZODB.MappingStorage
database = DB(ZODB.blob.BlobStorage(
'blobs', ZODB.MappingStorage.MappingStorage()))
connection = database.open()
root = connection.root()
transaction.begin()
root['blob'] = Blob()
transaction.commit()
stream = StringIO()
p = Pickler(stream, 1)
p.dump(root['blob'])
u = Unpickler(stream)
stream.seek(0)
clone = u.load()
clone._p_invalidate()
# it should also be possible to open the cloned blob
# (even though it won't contain the original data)
clone.open()
# tearDown
database.close()
def checkAutoReconnectOnSync(self):
# Verify auto-reconnect.
db = DB(self._storage)
try:
c1 = db.open()
r = c1.root()
c1._storage._load_conn.close()
c1._storage.sync()
# ZODB5 calls sync when a connection is opened. Our monkey
# patch on a Connection makes sure that works in earlier
# versions, but we don't have that patch on ZODB5. So test
# the storage directly. NOTE: The load connection must be open.
# to trigger the actual sync.
r = c1.root()
r['alpha'] = 1
transaction.commit()
c1.close()
c1._storage._load_conn.close()
c1._storage._store_conn.close()
c2 = db.open()
self.assertIs(c2, c1)
r = c2.root()
self.assertEqual(r['alpha'], 1)
r['beta'] = PersistentMapping()
c2.add(r['beta'])
transaction.commit()
c2.close()
finally:
db.close()
def testSimpleBlobRecovery(self):
if hasattr(ZODB.interfaces, 'IBlobStorageRestoreable'):
self.assert_(
ZODB.interfaces.IBlobStorageRestoreable.providedBy(
self._storage)
)
db = DB(self._storage)
conn = db.open()
conn.root()[1] = ZODB.blob.Blob()
transaction.commit()
conn.root()[2] = ZODB.blob.Blob()
conn.root()[2].open('w').write(b('some data'))
transaction.commit()
conn.root()[3] = ZODB.blob.Blob()
conn.root()[3].open('w').write(
(b('').join(struct.pack(">I", random.randint(0, (1<<32)-1))
for i in range(random.randint(10000,20000)))
)[:-random.randint(1,4)]
)
transaction.commit()
conn.root()[2] = ZODB.blob.Blob()
conn.root()[2].open('w').write(b('some other data'))
transaction.commit()
self._dst.copyTransactionsFrom(self._storage)
self.compare(self._storage, self._dst)
def testLargeBlob(self):
# Large blobs are chunked into multiple pieces, we want to know
# if they come out the same way they went in.
db = DB(self._storage)
conn = db.open()
blob = conn.root()[1] = ZODB.blob.Blob()
size = sizeof_fmt(self.testsize)
self._log('Creating %s blob file' % size)
blob_file = blob.open('w')
signature = random_file(self.testsize, blob_file)
blob_file.close()
self._log('Committing %s blob file' % size)
transaction.commit()
conn.close()
# Clear the cache
for base, _dir, files in os.walk('.'):
for f in files:
if f.endswith('.blob'):
ZODB.blob.remove_committed(os.path.join(base, f))
# Re-download blob
self._log('Caching %s blob file' % size)
conn = db.open()
with conn.root()[1].open('r') as blob:
self._log('Creating signature for %s blob cache' % size)
self.assertEqual(md5sum(blob), signature)
conn.close()
db.close()
def checkCrossConnectionInvalidation(self):
# Verify connections see updated state at txn boundaries
db = DB(self._storage)
try:
c1 = db.open()
r1 = c1.root()
r1['myobj'] = 'yes'
c2 = db.open()
r2 = c2.root()
self.assertNotIn('myobj', r2)
storage = c1._storage
t = transaction.Transaction()
t.description = u'invalidation test'
c1.tpc_begin(t)
c1.commit(t)
storage.tpc_vote(storage._transaction)
storage.tpc_finish(storage._transaction)
self.assertNotIn('myobj', r2)
c2.sync()
self.assertIn('myobj', r2)
self.assertEqual(r2['myobj'], 'yes')
finally:
db.close()
class BlobStorage(object):
def __init__(self):
"""Prepares for a functional test case.
"""
transaction.abort()
storage = DemoStorage("Demo Storage")
if not IBlobStorage.providedBy(storage):
raise RuntimeError
self.db = DB(storage)
self.connection = None
def clean(self):
"""Cleans up after a functional test case.
"""
transaction.abort()
if self.connection:
self.connection.close()
self.connection = None
self.db.close()
def close(self):
if self.connection:
self.connection.close()
self.connection = None
self.db.close()
def open(self):
if self.connection:
self.close()
self.connection = self.db.open()
return self.connection.root()
def checkAutoReconnect(self):
# Verify auto-reconnect
db = DB(self._storage)
try:
c1 = db.open()
r = c1.root()
r['alpha'] = 1
transaction.commit()
c1.close()
c1._storage._load_conn.close()
c1._storage._store_conn.close()
# ZODB5 implicitly calls sync
# immediately when a connection is opened;
# fake that here for older releases.
c2 = db.open()
self.assertIs(c2, c1)
c2.sync()
r = c2.root()
self.assertEqual(r['alpha'], 1)
r['beta'] = PersistentMapping()
c2.add(r['beta'])
transaction.commit()
c2.close()
finally:
db.close()
def _open_client (self, location):
"""Open 'location' (a (hostname, port_number) tuple) as a ZEO
ClientStorage, and then open a ZODB database around that. Return
a (database, connection) tuple.
We override dulcinea.database.ObjectDatabase._open_client to support
username/password.
"""
host, port = location
if host == "":
# If the specified hostname is the empty string, then
# 'localhost' is used.
location = ('localhost', port)
site_config = get_config()
username = site_config.get(self.site, 'zeo-username', fallback='')
password = site_config.get(self.site, 'zeo-password', fallback='')
# we use QonClientStorage instead of ClientStorage to:
# 1. workaround ClientStorage's cache_size bug
# 2. enable cache instrumentation (if qon.local.CACHE_INSTRUMENTATION is True)
from qon.cache_logging import QonClientStorage
self.storage = QonClientStorage(location,
var='/var/tmp',
wait=0,
cache_size=150*MB,
username=username,
password=password)
db = DB(self.storage)
return (db, db.open())
class DbAdapter:
def __init__(self, path="data.db"):
self.path = path
def connect(self):
self.storage = FileStorage(self.path)
self.db = DB(self.storage)
self.conn = self.db.open()
return self.conn.root()
def begin_transaction(self):
transaction.begin()
def commit(self):
transaction.commit()
def rollback(self):
transaction.abort()
def disconnect(self):
self.conn.close()
self.db.close()
self.storage.close()
if os.path.exists(self.path + ".lock"):
os.remove(self.path + ".lock")
class Storage(object):
def __init__(self):
"""Prepares for a functional test case.
"""
# we prevent any craziness here
transaction.abort()
storage = DemoStorage("Demo Storage")
self.db = DB(storage)
self.connection = None
def clean(self):
"""Cleans up after a functional test case.
"""
transaction.abort()
if self.connection:
self.connection.close()
self.connection = None
self.db.close()
def close(self):
if self.connection:
self.connection.close()
self.connection = None
self.db.close()
def open(self):
if self.connection:
self.close()
self.connection = self.db.open()
return self.connection.root()
def checkPackKeepNewObjects(self):
# Packing should not remove objects created or modified after
# the pack time, even if they are unreferenced.
db = DB(self._storage)
try:
# add some data to be packed
c = db.open()
extra1 = PersistentMapping()
c.add(extra1)
extra2 = PersistentMapping()
c.add(extra2)
transaction.commit()
# Choose the pack time
now = packtime = time.time()
while packtime <= now:
time.sleep(0.1)
packtime = time.time()
while packtime == time.time():
time.sleep(0.1)
extra2.foo = "bar"
extra3 = PersistentMapping()
c.add(extra3)
transaction.commit()
self._storage.pack(packtime, referencesf)
# extra1 should have been garbage collected
self.assertRaises(KeyError, self._storage.load, extra1._p_oid, "")
# extra2 and extra3 should both still exist
self._storage.load(extra2._p_oid, "")
self._storage.load(extra3._p_oid, "")
finally:
db.close()
def checkPackGC(self, expect_object_deleted=True, close=True):
db = DB(self._storage)
try:
c1 = db.open()
r1 = c1.root()
r1['alpha'] = PersistentMapping()
transaction.commit()
oid = r1['alpha']._p_oid
r1['alpha'] = None
transaction.commit()
# The object should still exist
self._storage.load(oid, '')
# Pack
now = packtime = time.time()
while packtime <= now:
packtime = time.time()
self._storage.pack(packtime, referencesf)
self._storage.sync()
if expect_object_deleted:
# The object should now be gone
self.assertRaises(KeyError, self._storage.load, oid, '')
else:
# The object should still exist
self._storage.load(oid, '')
finally:
if close:
db.close()
return oid
def checkNonASCIITransactionMetadata(self):
# Verify the database stores and retrieves non-ASCII text
# in transaction metadata.
ugly_string = ''.join(chr(c) for c in range(256))
if isinstance(ugly_string, bytes):
# Always text. Use latin 1 because it can decode any arbitrary
# bytes.
ugly_string = ugly_string.decode('latin-1')
# The storage layer is defined to take bytes (implicitly in
# older ZODB releases, explicitly in ZODB 5.something), but historically
# it can accept either text or bytes. However, it always returns bytes
check_string = ugly_string.encode("utf-8")
db = DB(self._storage)
try:
c1 = db.open()
r1 = c1.root()
r1['alpha'] = 1
transaction.get().setUser(ugly_string)
transaction.commit()
r1['alpha'] = 2
transaction.get().note(ugly_string)
transaction.commit()
info = self._storage.undoInfo()
self.assertEqual(info[0]['description'], check_string)
self.assertEqual(info[1]['user_name'], b'/ ' + check_string)
finally:
db.close()
def checkPackOldUnreferenced(self):
db = DB(self._storage)
try:
c1 = db.open()
r1 = c1.root()
r1['A'] = PersistentMapping()
B = PersistentMapping()
r1['A']['B'] = B
transaction.get().note('add A then add B to A')
transaction.commit()
del r1['A']['B']
transaction.get().note('remove B from A')
transaction.commit()
r1['A']['C'] = ''
transaction.get().note('add C to A')
transaction.commit()
now = packtime = time.time()
while packtime <= now:
packtime = time.time()
self._storage.pack(packtime, referencesf)
# B should be gone, since nothing refers to it.
self.assertRaises(KeyError, self._storage.load, B._p_oid, '')
finally:
db.close()
def test_pack_with_1_day(self):
from ZODB.DB import DB
from ZODB.FileStorage import FileStorage
from ZODB.POSException import POSKeyError
import time
import transaction
from relstorage.zodbpack import main
storage = FileStorage(self.db_fn, create=True)
db = DB(storage)
conn = db.open()
conn.root()['x'] = 1
transaction.commit()
oid = b('\0' * 8)
state, serial = storage.load(oid, b(''))
time.sleep(0.1)
conn.root()['x'] = 2
transaction.commit()
conn.close()
self.assertEqual(state, storage.loadSerial(oid, serial))
db.close()
storage = None
main(['', '--days=1', self.cfg_fn])
# packing should not have removed the old state.
storage = FileStorage(self.db_fn)
self.assertEqual(state, storage.loadSerial(oid, serial))
storage.close()
def database(db):
"""Load ZODB database from Python module or FileStorage file"""
if type(db) is str:
# Database name
if db.endswith('.py'):
# Python source, exec it
globals = {}
execfile(db, globals)
if 'DB' in globals:
db = globals['DB']
else:
storage = globals['Storage']
from ZODB.DB import DB
db = DB(storage, cache_size=4000)
elif db.endswith(".fs"):
from ZODB.FileStorage import FileStorage
from ZODB.DB import DB
storage = FileStorage(db)
db = DB(storage, cache_size=4000)
# The following will fail unless the application has been configured.
from zope.event import notify
notify(zope.processlifetime.DatabaseOpened(db))
return db
def checkConcurrentUpdates2StoragesMT(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1, 3)
t2 = self.StressThread(self, db2, stop, 2, cd, 2, 3, 0.01)
t3 = self.StressThread(self, db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
time.sleep(.1)
time.sleep(.1)
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
def testRedo(self):
database = DB(self._storage)
connection = database.open()
root = connection.root()
blob = Blob()
transaction.begin()
blob.open('w').write(b('this is state 1'))
root['blob'] = blob
transaction.commit()
transaction.begin()
blob = root['blob']
blob.open('w').write(b('this is state 2'))
transaction.commit()
database.undo(database.undoLog(0, 1)[0]['id'])
transaction.commit()
self.assertEqual(blob.open('r').read(), b('this is state 1'))
database.undo(database.undoLog(0, 1)[0]['id'])
transaction.commit()
self.assertEqual(blob.open('r').read(), b('this is state 2'))
database.close()
def checkConcurrentUpdates2Storages(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
storage2 = self.openClientStorage()
db2 = DB(storage2)
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run two threads that update the BTree
cd = {}
t1 = self.StressThread(self, db1, stop, 1, cd, 1)
t2 = self.StressThread(self, db2, stop, 2, cd, 2)
self.go(stop, cd, t1, t2)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
db2.close()
def zodb_conn_open(path): from ZODB.FileStorage import FileStorage from ZODB.DB import DB storage = FileStorage(path) db = DB(storage) connection = db.open() zodb = connection.root() return db,zodb
def create():
storage = FileStorage("Data.fs")
db = DB(storage)
connection = db.open()
root = connection.root()
root["p1"] = Person("john", 49)
root["p2"] = Person("nick", 35)
transaction.commit()
print root.keys()
def open_db():
storage = FileStorage("Data.fs")
db = DB(storage)
connection = db.open()
root = connection.root()
for k in root:
a = root[k]
print a.name
def setSystem(system): addr = 'localhost', 9001 storage = ClientStorage(addr) db = DB(storage) connection = db.open() root = connection.root() root["system"] = system transaction.commit() db.close()
def testLargeBlob(self):
# Large blobs are chunked into multiple pieces, we want to know
# if they come out the same way they went in.
db = DB(self._storage)
conn = db.open()
blob = conn.root()[1] = ZODB.blob.Blob()
size = sizeof_fmt(self.testsize)
self._log('Creating %s blob file' % size)
signature = random_file(self.testsize, blob.open('w'))
self._log('Committing %s blob file' % size)
transaction.commit()
# Clear the cache
for base, dir, files in os.walk('.'):
for f in files:
if f.endswith('.blob'):
ZODB.blob.remove_committed(os.path.join(base, f))
# Re-download blob
self._log('Caching %s blob file' % size)
conn = db.open()
blob = conn.root()[1].open('r')
self._log('Creating signature for %s blob cache' % size)
self.assertEqual(md5sum(blob), signature)
class CompareTest(unittest.TestCase):
s = "A string with hi-bit-set characters: \700\701"
u = u"A unicode string"
def setUp(self):
# These defaults only make sense if the default encoding
# prevents s from being promoted to Unicode.
self.assertRaises(UnicodeError, unicode, self.s)
# An object needs to be added to the database to
self.db = DB(MappingStorage())
root = self.db.open().root()
self.bucket = root["bucket"] = Bucket()
self.set = root["set"] = Set()
transaction.commit()
def tearDown(self):
self.assert_(self.bucket._p_changed != 2)
self.assert_(self.set._p_changed != 2)
transaction.abort()
def assertUE(self, callable, *args):
self.assertRaises(UnicodeError, callable, *args)
def testBucketGet(self):
self.bucket[self.s] = 1
self.assertUE(self.bucket.get, self.u)
def testSetGet(self):
self.set.insert(self.s)
self.assertUE(self.set.remove, self.u)
def testBucketSet(self):
self.bucket[self.s] = 1
self.assertUE(self.bucket.__setitem__, self.u, 1)
def testSetSet(self):
self.set.insert(self.s)
self.assertUE(self.set.insert, self.u)
def testBucketMinKey(self):
self.bucket[self.s] = 1
self.assertUE(self.bucket.minKey, self.u)
def testSetMinKey(self):
self.set.insert(self.s)
self.assertUE(self.set.minKey, self.u)
def _populate_root_and_mapping(self):
tx1 = transaction.TransactionManager()
storage1 = self._storage
db1 = self._closing(DB(storage1))
c1 = db1.open(tx1)
root = c1.root()
root.myobj1 = mapping = PersistentMapping()
root.myobj = 1
tx1.commit()
c1._storage._cache.clear(load_persistent=False)
c1._storage.poll_invalidations()
root.myobj = 2
tx1.commit()
c1._storage._cache.clear(load_persistent=False)
c1._storage.poll_invalidations()
root.myobj = 3
tx1.commit()
root_tid = self.assert_oid_known(ROOT_OID, c1)
c1._storage._cache.clear(load_persistent=False)
# Now, mutate an object that's not the root
# so that we get a new transaction after the root was
# modified. This transaction will be included in
# a persistent cache.
c1._storage.poll_invalidations()
root.myobj1.key = PersistentMapping()
mapping_oid = mapping._p_oid
mapping_oid_int = bytes8_to_int64(mapping_oid)
tx1.commit()
mapping_tid = self.assert_oid_known(mapping_oid_int, c1)
self.assert_checkpoints(c1, (root_tid, root_tid))
self.assert_oid_current(mapping_oid_int, c1)
# the root is not in a delta
self.assert_oid_not_known(ROOT_OID, c1)
# Nor is it in the cache, because the Connection's
# object cache still had the root and we were never
# asked.
self.assert_oid_not_cached(ROOT_OID, c1)
# So lets get it in the cache with its current TID.
c1._storage.load(z64)
self.assert_cached_exact(ROOT_OID, root_tid, c1)
c1.close()
return root_tid, mapping_tid, db1
def ZODBDatabaseConfigurationFactory(key, dbconfig):
config = dbconfig.get('configuration', {})
fs = ZODB.FileStorage.FileStorage(dbconfig['path'])
db = DB(fs)
try:
rootobj = db.open().root()
if not IDatabase.providedBy(rootobj):
alsoProvides(rootobj, IDatabase)
transaction.commit()
rootobj = None
except:
pass
finally:
db.close()
# Set request aware database for app
db = RequestAwareDB(dbconfig['path'], **config)
return Database(key, db)
def checkIsolationLevels(self):
def assert_storage(storage):
load_cur = storage._load_connection.cursor
store_cur = storage._store_connection.cursor
version_detector = storage._adapter.version_detector
if not version_detector.supports_transaction_isolation(load_cur):
raise unittest.SkipTest(
"Needs MySQL better than %s" %
(version_detector.get_version(load_cur)))
for cur, ex_iso, ex_ro, ex_timeout in (
# Timeout for load is mysql default.
[load_cur, 'REPEATABLE-READ', True, 50],
[
store_cur, 'READ-COMMITTED', False,
self.DEFAULT_COMMIT_LOCK_TIMEOUT
],
):
cur.execute("""
SELECT @@transaction_isolation,
@@transaction_read_only,
@@innodb_lock_wait_timeout
""")
row, = cur.fetchall()
iso, ro, timeout = row
__traceback_info__ = row
iso = iso.decode('ascii') if not isinstance(iso, str) else iso
self.assertEqual(iso, ex_iso)
self.assertEqual(ro, ex_ro)
self.assertEqual(timeout, ex_timeout)
# By default
assert_storage(self._storage)
# In a new instance, and after we do a transaction with it.
from ZODB.DB import DB
import transaction
db = self._closing(DB(self._storage))
conn = self._closing(db.open())
assert_storage(conn._storage)
conn.root()['obj'] = 1
transaction.commit()
assert_storage(conn._storage)
def __store_two_for_read_current_error(self):
db = self._closing(DB(self._storage))
conn = db.open()
root = conn.root()
root['object1'] = MinPO('object1')
root['object2'] = MinPO('object2')
transaction.commit()
obj1_oid = root['object1']._p_oid
obj2_oid = root['object2']._p_oid
obj1_tid = root['object1']._p_serial
assert obj1_tid == root['object2']._p_serial
conn.close()
# We can't close the DB, that will close the storage that we
# still need.
return obj1_oid, obj2_oid, obj1_tid, db
def checkPollInterval(self, shared_cache=True):
# Verify the poll_interval parameter causes RelStorage to
# delay invalidation polling.
self._storage = self.make_storage(poll_interval=3600,
share_local_cache=shared_cache)
db = DB(self._storage)
try:
tm1 = transaction.TransactionManager()
c1 = db.open(transaction_manager=tm1)
r1 = c1.root()
r1['alpha'] = 1
tm1.commit()
tm2 = transaction.TransactionManager()
c2 = db.open(transaction_manager=tm2)
r2 = c2.root()
self.assertEqual(r2['alpha'], 1)
self.assertFalse(c2._storage.need_poll())
self.assertTrue(c2._storage._poll_at > 0)
r1['alpha'] = 2
# commit c1 without committing c2.
tm1.commit()
if shared_cache:
# The cache reveals that a poll is needed even though
# the poll timeout has not expired.
self.assertTrue(c2._storage.need_poll())
tm2.commit()
r2 = c2.root()
self.assertEqual(r2['alpha'], 2)
self.assertFalse(c2._storage.need_poll())
else:
# The poll timeout has not expired, so no poll should occur
# yet, even after a commit.
self.assertFalse(c2._storage.need_poll())
tm2.commit()
r2 = c2.root()
self.assertEqual(r2['alpha'], 1)
# expire the poll timer and verify c2 sees the change
c2._storage._poll_at -= 3601
tm2.commit()
r2 = c2.root()
self.assertEqual(r2['alpha'], 2)
c2.close()
c1.close()
finally:
db.close()
def classFactory(self, connection, modulename, globalname):
if globalname in globalname_dict:
if globalname_dict[globalname][0]:
modulename = globalname_dict[globalname][0]
if globalname_dict[globalname][1]:
globalname = globalname_dict[globalname][1]
# There is an else in order to do not overwrite the rule.
# It could create some inconsistency
elif modulename in modulename_dict:
modulename = modulename_dict[modulename]
else:
# This is the case of a module with submodules
for former_mod_name in modulename_dict:
if former_mod_name[-1] == "." and modulename.startswith(
former_mod_name):
modulename = modulename_dict[former_mod_name] + modulename[
len(former_mod_name):]
return DB.classFactory(self, connection, modulename, globalname)
def testUndo(self):
database = DB(self._storage)
connection = database.open()
root = connection.root()
transaction.begin()
blob = Blob()
blob.open('w').write('this is state 1')
root['blob'] = blob
transaction.commit()
transaction.begin()
blob = root['blob']
blob.open('w').write('this is state 2')
transaction.commit()
database.undo(database.undoLog(0, 1)[0]['id'])
transaction.commit()
self.assertEqual(blob.open('r').read(), 'this is state 1')
database.close()
def checkConcurrentUpdates2Storages_emulated(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
storage2 = self.openClientStorage()
db2 = DB(storage2)
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
# DM: allow time for invalidations to come in and process them
time.sleep(0.1)
# Run two threads that update the BTree
t1 = StressTask(db1, 1, 1,)
t2 = StressTask(db2, 2, 2,)
_runTasks(100, t1, t2)
cn.sync()
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
db2.close()
def __store_two_for_read_current_error(self, release_extra_storage=False):
db = self._closing(DB(self._storage, pool_size=1))
conn = db.open()
root = conn.root()
root['object1'] = MinPO('object1')
root['object2'] = MinPO('object2')
transaction.commit()
obj1_oid = root['object1']._p_oid
obj2_oid = root['object2']._p_oid
obj1_tid = root['object1']._p_serial
assert obj1_tid == root['object2']._p_serial
conn.close()
# We can't close the DB, that will close the storage that we
# still need. But we can release its storage, since we'll never use
# this again.
if release_extra_storage:
conn._normal_storage.release()
return obj1_oid, obj2_oid, obj1_tid, db
def checkBTreesLengthStress(self):
# BTrees.Length objects are unusual Persistent objects: they
# set _p_independent and they frequently invoke conflict
# resolution. Run a stress test on them.
updates_per_thread = 50
thread_count = 4
from BTrees.Length import Length
db = DB(self._storage)
try:
c = db.open()
try:
c.root()['length'] = Length()
transaction.commit()
finally:
c.close()
def updater():
thread_db = DB(self._storage)
for i in range(updates_per_thread):
thread_c = thread_db.open()
try:
thread_c.root()['length'].change(1)
time.sleep(random.random() * 0.05)
transaction.commit()
finally:
thread_c.close()
import threading
threads = []
for i in range(thread_count):
t = threading.Thread(target=updater)
threads.append(t)
for t in threads:
t.start()
for t in threads:
t.join(120)
c = db.open()
try:
self.assertEqual(c.root()['length'](),
updates_per_thread * thread_count)
finally:
transaction.abort()
c.close()
finally:
db.close()
def testUndoAfterConsumption(self):
database = DB(self._storage)
connection = database.open()
root = connection.root()
transaction.begin()
open('consume1', 'wb').write(b('this is state 1'))
blob = Blob()
blob.consumeFile('consume1')
root['blob'] = blob
transaction.commit()
transaction.begin()
blob = root['blob']
open('consume2', 'wb').write(b('this is state 2'))
blob.consumeFile('consume2')
transaction.commit()
database.undo(database.undoLog(0, 1)[0]['id'])
transaction.commit()
self.assertEqual(blob.open('r').read(), b('this is state 1'))
database.close()
def checkHistoricalConnection(self):
import datetime
import persistent
import ZODB.POSException
db = DB(self._storage)
conn = db.open()
root = conn.root()
root['first'] = persistent.mapping.PersistentMapping(count=0)
transaction.commit()
time.sleep(.02)
now = datetime.datetime.utcnow()
time.sleep(.02)
root['second'] = persistent.mapping.PersistentMapping()
root['first']['count'] += 1
transaction.commit()
transaction1 = transaction.TransactionManager()
historical_conn = db.open(transaction_manager=transaction1, at=now)
eq = self.assertEqual
# regular connection sees present:
eq(sorted(conn.root().keys()), ['first', 'second'])
eq(conn.root()['first']['count'], 1)
# historical connection sees past:
eq(sorted(historical_conn.root().keys()), ['first'])
eq(historical_conn.root()['first']['count'], 0)
# Can't change history:
historical_conn.root()['first']['count'] += 1
eq(historical_conn.root()['first']['count'], 1)
self.assertRaises(ZODB.POSException.ReadOnlyHistoryError,
transaction1.commit)
transaction1.abort()
eq(historical_conn.root()['first']['count'], 0)
historical_conn.close()
conn.close()
db.close()
def classFactory(self, connection, modulename, globalname):
# indico_ are new plugins, we don't need/want this kind of migration there
# The main reason for this check is not having to rename FoundationSyncTask
# which was deleted from core and now resides in a plugin.
if globalname in globalname_dict and not modulename.startswith(
'indico_'):
if globalname_dict[globalname][0]:
modulename = globalname_dict[globalname][0]
if globalname_dict[globalname][1]:
globalname = globalname_dict[globalname][1]
# There is an else in order to do not overwrite the rule.
# It could create some inconsistency
elif modulename in modulename_dict:
modulename = modulename_dict[modulename]
else:
# This is the case of a module with submodules
for former_mod_name in modulename_dict:
if former_mod_name[-1] == "." and modulename.startswith(
former_mod_name):
modulename = modulename_dict[former_mod_name] + modulename[
len(former_mod_name):]
return DB.classFactory(self, connection, modulename, globalname)
def setUp(self):
super(BasePublicationTests, self).setUp()
from zope.security.management import endInteraction
endInteraction()
ztapi.provideAdapter(IHTTPRequest, IUserPreferredCharsets,
HTTPCharsets)
self.policy = setSecurityPolicy(
simplepolicies.PermissiveSecurityPolicy)
self.storage = DemoStorage('test_storage')
self.db = db = DB(self.storage)
ztapi.provideUtility(IAuthentication, principalRegistry)
connection = db.open()
root = connection.root()
app = getattr(root, ZopePublication.root_name, None)
if app is None:
from zope.app.folder import rootFolder
app = rootFolder()
root[ZopePublication.root_name] = app
transaction.commit()
connection.close()
self.app = app
from zope.traversing.namespace import view, resource, etc
ztapi.provideNamespaceHandler('view', view)
ztapi.provideNamespaceHandler('resource', resource)
ztapi.provideNamespaceHandler('etc', etc)
self.request = TestRequest('/f1/f2')
self.user = Principal('test.principal')
self.request.setPrincipal(self.user)
from zope.interface import Interface
self.presentation_type = Interface
self.request._presentation_type = self.presentation_type
self.object = object()
self.publication = ZopePublication(self.db)
def checkDoubleCommitter(self):
# Verify we can store an object that gets committed twice in
# a single transaction.
db = DB(self._storage)
try:
conn = db.open()
try:
conn.root()['dc'] = DoubleCommitter()
transaction.commit()
conn2 = db.open()
self.assertEquals(conn2.root()['dc'].new_attribute, 1)
conn2.close()
finally:
transaction.abort()
conn.close()
finally:
db.close()
def __set_keys_in_root_to(self,
storage,
new_data,
old_tids,
old_data,
pack=False):
"""
Set the values for *new_data* in the root object of the storage.
And return the transaction ID of when we mad the change,
and the transaction ID of the last time the root changed.
Uses an independent transaction.
*old_tids* is a map from the keys in *new_data* to an expected TID
that should be cached. *old_value* is the same for the expected
current values on the root.
Closes *storage*.
"""
db1 = self._closing(DB(storage))
tx = transaction.TransactionManager()
c1 = db1.open(tx)
# We've polled and gained checkpoints
# self.assert_checkpoints(c1)
root = c1.root()
self.__do_sets(root, new_data, old_tids, old_data)
tx.commit()
self.__do_check_tids(root, old_tids)
tid_int = bytes8_to_int64(c1._storage.lastTransaction())
self.assertEqual(c1._storage._cache.current_tid, tid_int)
c1.close()
if pack:
storage.pack(tid_int, referencesf)
db1.close()
return tid_int, bytes8_to_int64(root._p_serial)
def main(args=None, start_serving=True):
logging.basicConfig(format="%(message)s")
options = parse_args(args)
db = open_db(options)
internal_db = DB(MappingStorage())
configure(options)
provideUtility(db, IDatabase, name='<target>')
notify(zope.app.appsetup.interfaces.DatabaseOpened(internal_db))
start_server(options, internal_db)
notify(zope.app.appsetup.interfaces.ProcessStarting())
install_provides_hack()
if start_serving:
serve_forever()
def stackDemoStorage(db=None, name=None):
"""Create a new DemoStorage that has the given database as a base.
``db`` may be none, in which case a base demo storage will be created.
``name`` is optional, but can be used to name the storage.
The usual pattern in a layer is::
def setUp(self):
self['zodbDB'] = stackDemoStorage(self.get('zodbDB'), name='mylayer') # noqa
def tearDown(self):
self['zodbDB'].close()
del self['zodbDB']
"""
from ZODB.DemoStorage import DemoStorage
from ZODB.DB import DB
if db is not None:
storage = DemoStorage(name=name, base=db.storage)
else:
storage = DemoStorage(name=name)
return DB(storage)
def checkCrossConnectionInvalidation(self):
# Verify connections see updated state at txn boundaries.
# This will fail if the Connection doesn't poll for changes.
db = DB(self._storage)
try:
c1 = db.open(transaction.TransactionManager())
r1 = c1.root()
r1['myobj'] = 'yes'
c2 = db.open(transaction.TransactionManager())
r2 = c2.root()
self.assertTrue('myobj' not in r2)
c1.transaction_manager.commit()
self.assertTrue('myobj' not in r2)
c2.sync()
self.assertTrue('myobj' in r2)
self.assertTrue(r2['myobj'] == 'yes')
finally:
db.close()
def checkConcurrentUpdates1Storage(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run two threads that update the BTree
cd = {}
t1 = self.StressThread(db1, stop, 1, cd, 1, sleep=0.01)
t2 = self.StressThread(db1, stop, 2, cd, 2, sleep=0.01)
self.go(stop, cd, t1, t2)
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
def checkAutoReconnect(self):
# Verify auto-reconnect
db = DB(self._storage)
try:
c1 = db.open()
r = c1.root()
r['alpha'] = 1
transaction.commit()
c1.close()
c1._storage._load_conn.close()
c1._storage._store_conn.close()
c2 = db.open()
self.assert_(c2 is c1)
r = c2.root()
self.assertEqual(r['alpha'], 1)
r['beta'] = PersistentMapping()
c2.add(r['beta'])
transaction.commit()
c2.close()
finally:
db.close()
def openDB(self):
from ZODB.FileStorage import FileStorage
from ZODB.DB import DB
n = 'fs_tmp__%s' % os.getpid()
self.storage = FileStorage(n)
self.db = DB(self.storage)
def __init__(self, filename=None, **argsdict):
self.roots = None
self.propsGrid = None
self.Modified = False
self.Initial = False
self.db_path = None
self.layers = {}
self.Filename = filename
self.wfs = tempfile.mkdtemp(suffix='.deca')
if filename is not None:
# open filesystem
try:
f = zipfile.ZipFile(self.Filename)
f.extractall(self.wfs)
f.close()
self.db_path = os.path.join(self.wfs, 'database')
self.db_path = os.path.join(self.db_path, 'filestorage.sampo')
if not os.path.exists(self.db_path):
self.db_path = None
except Exception:
self.db_path = None
if self.db_path is None:
# create new world filesystem if file not given, or error occurred
if filename is None:
self.Initial = True
try:
# create initial structure
os.makedirs(os.path.join(self.wfs, 'profiles'))
os.makedirs(self.PixmapsPath)
os.makedirs(os.path.join(self.wfs, 'database'))
os.makedirs(self.EnginesPath)
os.makedirs(
os.path.join(self.EnginesPath,
'Default')) # engines for 'Default' template
os.makedirs(os.path.join(self.EnginesPath,
'layer')) # engines for layer
os.makedirs(self.ReportsPath)
os.makedirs(self.ResultsPath)
os.makedirs(self.AttachmentsPath)
self.db_path = os.path.join(self.wfs, 'database')
self.db_path = os.path.join(self.db_path, 'filestorage.sampo')
cname = os.path.join(ed_glob.CONFIG['GLOBAL_PROFILE_DIR'],
'default.ppb')
f = open(cname, 'rb')
store = open(
os.path.join(self.wfs,
'profiles' + os.sep + 'default.ppb'), 'wb')
store.write(f.read())
store.close()
except Exception as cond:
wx.MessageBox(
_("Can't create world's storage!\n%s\nExiting...") % cond,
_("Sampo Framework"), wx.OK | wx.CENTER | wx.ICON_ERROR)
wx.GetApp().Exit()
# initialize repository filters
if not os.path.exists(os.path.join(self.wfs, '.hgignore')):
f = open(os.path.join(self.wfs, '.hgignore'), 'w')
f.write('syntax: glob\n')
f.write('.hg-*\n')
f.write('profiles\n')
f.write('database\n')
f.write('results\n')
f.write('attachments\n')
f.write('*.pyc\n')
f.write('*.pyo\n')
f.close()
self._repo = HgCommon.HgCreate(self.wfs, **argsdict)
# initialize database
ed_glob.CONFIG['PROFILE_DIR'] = os.path.join(self.wfs, 'profiles')
# todo: load profile value and swich to remote DB if necessary
# drop database indexes if platrorm doesn't match with previous save
try:
saved_platform = ''
try:
f = open(
os.path.join(self.wfs,
'profiles' + os.sep + 'platform.dat'), 'r')
saved_platform = f.read()
f.close()
except Exception:
pass
if saved_platform != platform.system():
os.unlink(self.db_path + '.index')
os.unlink(self.db_path + '.tmp')
except Exception:
pass
try:
self.storage = FileStorage(self.db_path)
except Exception:
# try to remove indexes and reload base
try:
os.unlink(self.db_path + '.index')
os.unlink(self.db_path + '.tmp')
os.unlink(self.db_path + '.lock')
except Exception:
pass
self.storage = FileStorage(self.db_path)
self.odb = DB(self.storage)
self.connection = self.odb.open()
self.roots = self.connection.root()
self.transaction = self.connection.transaction_manager.begin()
# always create object's repository
if self.Initial:
self.roots[self.ID_Repository] = DecaRepoStorage()
self.roots[self.ID_Configuration] = PersistentMapping()
self.layers[self.ID_Repository] = DecaRepo(self, self.ID_Repository)
def checkConcurrentUpdates2Storages(self):
self._storage = storage1 = self.openClientStorage()
storage2 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(storage2)
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run two threads that update the BTree
cd = {}
t1 = self.StressThread(db1, stop, 1, cd, 1)
t2 = self.StressThread(db2, stop, 2, cd, 2)
self.go(stop, cd, t1, t2)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2)
cn.close()
db1.close()
db2.close()
def xxxcheckConcurrentUpdatesInVersions(self):
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = VersionStressThread(db1, stop, 1, cd, 1, 3)
t2 = VersionStressThread(db2, stop, 2, cd, 2, 3, 0.01)
t3 = VersionStressThread(db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
def checkConcurrentLargeUpdates(self):
# Use 3 threads like the 2StorageMT test above.
self._storage = storage1 = self.openClientStorage()
db1 = DB(storage1)
db2 = DB(self.openClientStorage())
stop = threading.Event()
cn = db1.open()
tree = cn.root()["tree"] = OOBTree()
for i in range(0, 3000, 2):
tree[i] = 0
transaction.commit()
cn.close()
# Run three threads that update the BTree.
# Two of the threads share a single storage so that it
# is possible for both threads to read the same object
# at the same time.
cd = {}
t1 = LargeUpdatesThread(db1, stop, 1, cd, 1, 3, 0.02)
t2 = LargeUpdatesThread(db2, stop, 2, cd, 2, 3, 0.01)
t3 = LargeUpdatesThread(db2, stop, 3, cd, 3, 3, 0.01)
self.go(stop, cd, t1, t2, t3)
while db1.lastTransaction() != db2.lastTransaction():
db1._storage.sync()
db2._storage.sync()
cn = db1.open()
tree = cn.root()["tree"]
self._check_tree(cn, tree)
# Purge the tree of the dummy entries mapping to 0.
losers = [k for k, v in tree.items() if v == 0]
for k in losers:
del tree[k]
transaction.commit()
self._check_threads(tree, t1, t2, t3)
cn.close()
db1.close()
db2.close()
class TestTraverse(unittest.TestCase):
def setUp(self):
import io
import transaction
from AccessControl import SecurityManager
from AccessControl.SecurityManagement import newSecurityManager
from OFS.Application import Application
from OFS.Folder import manage_addFolder
from OFS.Image import manage_addFile
from Testing.makerequest import makerequest
from ZODB.DB import DB
from ZODB.DemoStorage import DemoStorage
s = DemoStorage()
self.connection = DB(s).open()
try:
r = self.connection.root()
a = Application()
r['Application'] = a
self.root = a
responseOut = self.responseOut = io.BytesIO()
self.app = makerequest(self.root, stdout=responseOut)
manage_addFolder(self.app, 'folder1')
folder1 = getattr(self.app, 'folder1')
setattr(folder1, '+something', 'plus')
folder1.all_meta_types = ({
'name': 'File',
'action': 'manage_addFile',
'permission': 'Add images and files'
}, )
manage_addFile(folder1,
'file',
file=b'',
content_type='text/plain')
# Hack, we need a _p_mtime for the file, so we make sure that it
# has one. We use a subtransaction, which means we can rollback
# later and pretend we didn't touch the ZODB.
transaction.commit()
except Exception:
self.connection.close()
raise
transaction.begin()
self.folder1 = getattr(self.app, 'folder1')
self.policy = UnitTestSecurityPolicy()
self.oldPolicy = SecurityManager.setSecurityPolicy(self.policy)
newSecurityManager(None, self._makeUser().__of__(self.root))
def tearDown(self):
import transaction
self._setupSecurity()
del self.oldPolicy
del self.policy
del self.folder1
transaction.abort()
self.app._p_jar.sync()
self.connection.close()
del self.app
del self.responseOut
del self.root
del self.connection
def _makeUser(self):
from Acquisition import Implicit
class UnitTestUser(Implicit):
"""
Stubbed out manager for unit testing purposes.
"""
def getId(self):
return 'unit_tester'
getUserName = getId
def allowed(self, object, object_roles=None):
return 1
return UnitTestUser()
def _makeBoboTraversable(self):
from OFS.SimpleItem import SimpleItem
class BoboTraversable(SimpleItem):
__allow_access_to_unprotected_subobjects__ = 1
def __bobo_traverse__(self, request, name):
if name == 'bb_subitem':
return BoboTraversable().__of__(self)
elif name == 'bb_method':
return self.bb_method
elif name == 'bb_status':
return self.bb_status
elif name == 'manufactured':
return 42
else:
raise KeyError
def bb_method(self):
"""Test Method"""
pass
bb_status = 'screechy'
return BoboTraversable()
def _makeBoboTraversableWithAcquisition(self):
from OFS.SimpleItem import SimpleItem
class BoboTraversableWithAcquisition(SimpleItem):
""" A BoboTraversable which may use acquisition to find objects.
This is similar to how the __bobo_traverse__ behaves).
"""
def __bobo_traverse__(self, request, name):
from Acquisition import aq_get
return aq_get(self, name)
return BoboTraversableWithAcquisition()
def _makeRestricted(self, name='dummy'):
from OFS.SimpleItem import SimpleItem
class Restricted(SimpleItem):
"""Instance we'll check with ProtectedMethodSecurityPolicy
"""
getId__roles__ = None # ACCESS_PUBLIC
def getId(self):
return self.id
private__roles__ = () # ACCESS_PRIVATE
def private(self):
return 'private!'
# not protected
def ohno(self):
return 'ohno!'
return Restricted(name)
def _setupSecurity(self, policy=None):
from AccessControl import SecurityManager
from AccessControl.SecurityManagement import noSecurityManager
if policy is None:
policy = self.oldPolicy
noSecurityManager()
SecurityManager.setSecurityPolicy(policy)
def test_interfaces(self):
from OFS.interfaces import ITraversable
from OFS.Traversable import Traversable
from zope.interface.verify import verifyClass
verifyClass(ITraversable, Traversable)
def testTraversePath(self):
self.assertTrue('file' in self.folder1.objectIds())
self.assertTrue(
self.folder1.unrestrictedTraverse(('', 'folder1', 'file')))
self.assertTrue(self.folder1.unrestrictedTraverse(('', 'folder1')))
def testTraverseURLNoSlash(self):
self.assertTrue('file' in self.folder1.objectIds())
self.assertTrue(self.folder1.unrestrictedTraverse('/folder1/file'))
self.assertTrue(self.folder1.unrestrictedTraverse('/folder1'))
def testTraverseURLSlash(self):
self.assertTrue('file' in self.folder1.objectIds())
self.assertTrue(self.folder1.unrestrictedTraverse('/folder1/file/'))
self.assertTrue(self.folder1.unrestrictedTraverse('/folder1/'))
def testTraverseToNone(self):
self.assertRaises(KeyError, self.folder1.unrestrictedTraverse,
('', 'folder1', 'file2'))
self.assertRaises(KeyError, self.folder1.unrestrictedTraverse,
'/folder1/file2')
self.assertRaises(KeyError, self.folder1.unrestrictedTraverse,
'/folder1/file2/')
def testTraverseMethodRestricted(self):
from AccessControl import Unauthorized
self.root.my = self._makeRestricted('my')
my = self.root.my
my.id = 'my'
self._setupSecurity(ProtectedMethodSecurityPolicy())
r = my.restrictedTraverse('getId')
self.assertEqual(r(), 'my')
self.assertRaises(Unauthorized, my.restrictedTraverse, 'private')
self.assertRaises(Unauthorized, my.restrictedTraverse, 'ohno')
def testBoboTraverseToWrappedSubObj(self):
# Verify it's possible to use __bobo_traverse__ with the
# Zope security policy.
self._setupSecurity()
bb = self._makeBoboTraversable()
self.assertRaises(KeyError, bb.restrictedTraverse, 'notfound')
bb.restrictedTraverse('bb_subitem')
def testBoboTraverseToMethod(self):
# Verify it's possible to use __bobo_traverse__ to a method.
self._setupSecurity()
bb = self._makeBoboTraversable()
self.assertTrue(bb.restrictedTraverse('bb_method') is not bb.bb_method)
def testBoboTraverseToSimpleAttrValue(self):
# Verify it's possible to use __bobo_traverse__ to a simple
# python value
self._setupSecurity()
bb = self._makeBoboTraversable()
self.assertEqual(bb.restrictedTraverse('bb_status'), 'screechy')
def testBoboTraverseToNonAttrValue(self):
# Verify it's possible to use __bobo_traverse__ to an
# arbitrary manufactured object
# Default security policy always seems to deny in this case, which
# is fine, but to test the code branch we sub in the forgiving one
self._setupSecurity(UnitTestSecurityPolicy())
bb = self._makeBoboTraversable()
self.assertTrue(bb.restrictedTraverse('manufactured') is 42)
def testBoboTraverseToAcquiredObject(self):
# Verify it's possible to use a __bobo_traverse__ which retrieves
# objects by acquisition
from Acquisition import aq_inner
self._setupSecurity()
bb = self._makeBoboTraversableWithAcquisition()
bb = bb.__of__(self.root)
self.assertEqual(bb.restrictedTraverse('folder1'), bb.folder1)
self.assertEqual(aq_inner(bb.restrictedTraverse('folder1')),
self.root.folder1)
def testBoboTraverseToAcquiredProtectedObject(self):
# Verify it's possible to use a __bobo_traverse__ which retrieves
# objects by acquisition
from AccessControl import Unauthorized
from AccessControl.Permissions import access_contents_information
self._setupSecurity()
folder = self.root.folder1
# restrict the ability to access the retrieved object itself
folder.manage_permission(access_contents_information, [], 0)
bb = self._makeBoboTraversableWithAcquisition()
bb = bb.__of__(self.root)
self.assertRaises(Unauthorized, bb.restrictedTraverse, 'folder1')
def testBoboTraverseToAcquiredAttribute(self):
# Verify it's possible to use __bobo_traverse__ to an acquired
# attribute
self._setupSecurity()
folder = self.root.folder1
folder.stuff = 'stuff here'
bb = self._makeBoboTraversableWithAcquisition()
bb = bb.__of__(folder)
self.assertEqual(bb.restrictedTraverse('stuff'), 'stuff here')
def testBoboTraverseToAcquiredProtectedAttribute(self):
# Verify that using __bobo_traverse__ to get an acquired but
# protected attribute results in Unauthorized
from AccessControl import Unauthorized
from AccessControl.Permissions import access_contents_information
self._setupSecurity()
folder = self.root.folder1
# We protect the the attribute by restricting access to the parent
folder.manage_permission(access_contents_information, [], 0)
folder.stuff = 'stuff here'
bb = self._makeBoboTraversableWithAcquisition()
bb = bb.__of__(folder)
self.assertRaises(Unauthorized, self.root.folder1.restrictedTraverse,
'stuff')
def testBoboTraverseTraversalDefault(self):
from OFS.SimpleItem import SimpleItem
from ZPublisher.interfaces import UseTraversalDefault
class BoboTraversableUseTraversalDefault(SimpleItem):
"""
A BoboTraversable class which may use "UseTraversalDefault"
(dependent on "name") to indicate that standard traversal should
be used.
"""
default = 'Default'
def __bobo_traverse__(self, request, name):
if name == 'normal':
return 'Normal'
raise UseTraversalDefault
bb = BoboTraversableUseTraversalDefault()
# normal access -- no traversal default used
self.assertEqual(bb.unrestrictedTraverse('normal'), 'Normal')
# use traversal default
self.assertEqual(bb.unrestrictedTraverse('default'), 'Default')
# test traversal default with acqires attribute
si = SimpleItem()
si.default_acquire = 'Default_Acquire'
si.bb = bb
self.assertEqual(si.unrestrictedTraverse('bb/default_acquire'),
'Default_Acquire')
def testAcquiredAttributeDenial(self):
# Verify that restrictedTraverse raises the right kind of exception
# on denial of access to an acquired attribute. If it raises
# AttributeError instead of Unauthorized, the user may never
# be prompted for HTTP credentials.
from AccessControl import Unauthorized
from AccessControl.SecurityManagement import newSecurityManager
self._setupSecurity(CruelSecurityPolicy())
newSecurityManager(None, self._makeUser().__of__(self.root))
self.root.stuff = 'stuff here'
self.assertRaises(Unauthorized, self.app.folder1.restrictedTraverse,
'stuff')
def testDefaultValueWhenUnathorized(self):
# Test that traversing to an unauthorized object returns
# the default when provided
from AccessControl.SecurityManagement import newSecurityManager
self._setupSecurity(CruelSecurityPolicy())
newSecurityManager(None, self._makeUser().__of__(self.root))
self.root.stuff = 'stuff here'
self.assertEqual(self.root.folder1.restrictedTraverse('stuff', 42), 42)
def testNotFoundIsRaised(self):
from OFS.SimpleItem import SimpleItem
from zExceptions import NotFound
from operator import getitem
self.folder1._setObject('foo', SimpleItem('foo'))
self.assertRaises(AttributeError, getitem, self.folder1.foo,
'doesntexist')
self.assertRaises(NotFound, self.folder1.unrestrictedTraverse,
'foo/doesntexist')
self.assertRaises(TypeError, getitem,
self.folder1.foo.isPrincipiaFolderish, 'doesntexist')
self.assertRaises(NotFound, self.folder1.unrestrictedTraverse,
'foo/isPrincipiaFolderish/doesntexist')
def testDefaultValueWhenNotFound(self):
# Test that traversing to a non-existent object returns
# the default when provided
self._setupSecurity()
self.assertEqual(self.root.restrictedTraverse('happy/happy', 'joy'),
'joy')
def testTraverseUp(self):
# Test that we can traverse upwards
from Acquisition import aq_base
self.assertTrue(
aq_base(self.root.folder1.file.restrictedTraverse('../..')) is
aq_base(self.root))
def testTraverseToNameStartingWithPlus(self):
# Verify it's possible to traverse to a name such as +something
self.assertTrue(
self.folder1.unrestrictedTraverse('+something') is 'plus')
def checkCrossConnectionIsolation(self):
# Verify MVCC isolates connections
db = DB(self._storage)
try:
c1 = db.open()
r1 = c1.root()
r1['alpha'] = PersistentMapping()
r1['gamma'] = PersistentMapping()
transaction.commit()
# Open a second connection but don't load root['alpha'] yet
c2 = db.open()
r2 = c2.root()
r1['alpha']['beta'] = 'yes'
storage = c1._storage
t = transaction.Transaction()
t.description = 'isolation test 1'
storage.tpc_begin(t)
c1.commit(t)
storage.tpc_vote(t)
storage.tpc_finish(t)
# The second connection will now load root['alpha'], but due to
# MVCC, it should continue to see the old state.
self.assert_(r2['alpha']._p_changed is None) # A ghost
self.assert_(not r2['alpha'])
self.assert_(r2['alpha']._p_changed == 0)
# make root['alpha'] visible to the second connection
c2.sync()
# Now it should be in sync
self.assert_(r2['alpha']._p_changed is None) # A ghost
self.assert_(r2['alpha'])
self.assert_(r2['alpha']._p_changed == 0)
self.assert_(r2['alpha']['beta'] == 'yes')
# Repeat the test with root['gamma']
r1['gamma']['delta'] = 'yes'
storage = c1._storage
t = transaction.Transaction()
t.description = 'isolation test 2'
storage.tpc_begin(t)
c1.commit(t)
storage.tpc_vote(t)
storage.tpc_finish(t)
# The second connection will now load root[3], but due to MVCC,
# it should continue to see the old state.
self.assert_(r2['gamma']._p_changed is None) # A ghost
self.assert_(not r2['gamma'])
self.assert_(r2['gamma']._p_changed == 0)
# make root[3] visible to the second connection
c2.sync()
# Now it should be in sync
self.assert_(r2['gamma']._p_changed is None) # A ghost
self.assert_(r2['gamma'])
self.assert_(r2['gamma']._p_changed == 0)
self.assert_(r2['gamma']['delta'] == 'yes')
finally:
db.close()
