def checkPackVersionReachable(self):
db = DB(self._storage)
cn = db.open()
root = cn.root()
names = "a", "b", "c"
for name in names:
root[name] = MinPO(name)
transaction.commit()
for name in names:
cn2 = db.open(version=name)
rt2 = cn2.root()
obj = rt2[name]
obj.value = MinPO("version")
transaction.commit()
cn2.close()
root["d"] = MinPO("d")
transaction.commit()
snooze()
self._storage.pack(time.time(), referencesf)
cn.sync()
# make sure all the non-version data is there
for name, obj in root.items():
self.assertEqual(name, obj.value)
# make sure all the version-data is there,
# and create a new revision in the version
for name in names:
cn2 = db.open(version=name)
rt2 = cn2.root()
obj = rt2[name].value
self.assertEqual(obj.value, "version")
obj.value = "still version"
transaction.commit()
cn2.close()
db.abortVersion("b")
txn = transaction.get()
txn.note("abort version b")
txn.commit()
t = time.time()
snooze()
L = db.undoInfo()
db.undo(L[0]["id"])
txn = transaction.get()
txn.note("undo abort")
txn.commit()
self._storage.pack(t, referencesf)
cn2 = db.open(version="b")
rt2 = cn2.root()
self.assertEqual(rt2["b"].value.value, "still version")
def checkTransactionalUndoAfterPackWithObjectUnlinkFromRoot(self):
eq = self.assertEqual
db = DB(self._storage)
conn = db.open()
try:
root = conn.root()
o1 = C()
o2 = C()
root['obj'] = o1
o1.obj = o2
txn = transaction.get()
txn.note(u'o1 -> o2')
txn.commit()
now = packtime = time.time()
while packtime <= now:
packtime = time.time()
o3 = C()
o2.obj = o3
txn = transaction.get()
txn.note(u'o1 -> o2 -> o3')
txn.commit()
o1.obj = o3
txn = transaction.get()
txn.note(u'o1 -> o3')
txn.commit()
log = self._storage.undoLog()
eq(len(log), 4)
for entry in zip(log, (b'o1 -> o3', b'o1 -> o2 -> o3', b'o1 -> o2',
b'initial database creation')):
eq(entry[0]['description'], entry[1])
self._storage.pack(packtime, referencesf)
log = self._storage.undoLog()
for entry in zip(log, (b'o1 -> o3', b'o1 -> o2 -> o3')):
eq(entry[0]['description'], entry[1])
tid = log[0]['id']
db.undo(tid)
txn = transaction.get()
txn.note(u'undo')
txn.commit()
# undo does a txn-undo, but doesn't invalidate
conn.sync()
log = self._storage.undoLog()
for entry in zip(log, (b'undo', b'o1 -> o3', b'o1 -> o2 -> o3')):
eq(entry[0]['description'], entry[1])
eq(o1.obj, o2)
eq(o1.obj.obj, o3)
self._iterate()
finally:
conn.close()
db.close()
def checkPackAfterUndoDeletion(self):
db = DB(self._storage)
cn = db.open()
try:
root = cn.root()
pack_times = []
def set_pack_time():
pack_times.append(time.time())
snooze()
root["key0"] = MinPO(0)
root["key1"] = MinPO(1)
root["key2"] = MinPO(2)
txn = transaction.get()
txn.note(u"create 3 keys")
txn.commit()
set_pack_time()
del root["key1"]
txn = transaction.get()
txn.note(u"delete 1 key")
txn.commit()
set_pack_time()
root._p_deactivate()
cn.sync()
self.assertTrue(listeq(root.keys(), ["key0", "key2"]))
L = db.undoInfo()
db.undo(L[0]["id"])
txn = transaction.get()
txn.note(u"undo deletion")
txn.commit()
set_pack_time()
root._p_deactivate()
cn.sync()
self.assertTrue(listeq(root.keys(), ["key0", "key1", "key2"]))
for t in pack_times:
self._storage.pack(t, referencesf)
root._p_deactivate()
cn.sync()
self.assertTrue(listeq(root.keys(), ["key0", "key1", "key2"]))
for i in range(3):
obj = root["key%d" % i]
self.assertEqual(obj.value, i)
root.items()
self._inter_pack_pause()
finally:
cn.close()
db.close()
def checkTransactionalUndoAfterPackWithObjectUnlinkFromRoot(self):
eq = self.assertEqual
db = DB(self._storage)
conn = db.open()
root = conn.root()
o1 = C()
o2 = C()
root['obj'] = o1
o1.obj = o2
txn = transaction.get()
txn.note(u'o1 -> o2')
txn.commit()
now = packtime = time.time()
while packtime <= now:
packtime = time.time()
o3 = C()
o2.obj = o3
txn = transaction.get()
txn.note(u'o1 -> o2 -> o3')
txn.commit()
o1.obj = o3
txn = transaction.get()
txn.note(u'o1 -> o3')
txn.commit()
log = self._storage.undoLog()
eq(len(log), 4)
for entry in zip(log, (b'o1 -> o3', b'o1 -> o2 -> o3',
b'o1 -> o2', b'initial database creation')):
eq(entry[0]['description'], entry[1])
self._storage.pack(packtime, referencesf)
log = self._storage.undoLog()
for entry in zip(log, (b'o1 -> o3', b'o1 -> o2 -> o3')):
eq(entry[0]['description'], entry[1])
tid = log[0]['id']
db.undo(tid)
txn = transaction.get()
txn.note(u'undo')
txn.commit()
# undo does a txn-undo, but doesn't invalidate
conn.sync()
log = self._storage.undoLog()
for entry in zip(log, (b'undo', b'o1 -> o3', b'o1 -> o2 -> o3')):
eq(entry[0]['description'], entry[1])
eq(o1.obj, o2)
eq(o1.obj.obj, o3)
self._iterate()
def checkPackAfterUndoDeletion(self):
db = DB(self._storage)
cn = db.open()
root = cn.root()
pack_times = []
def set_pack_time():
pack_times.append(time.time())
snooze()
root["key0"] = MinPO(0)
root["key1"] = MinPO(1)
root["key2"] = MinPO(2)
txn = transaction.get()
txn.note(u"create 3 keys")
txn.commit()
set_pack_time()
del root["key1"]
txn = transaction.get()
txn.note(u"delete 1 key")
txn.commit()
set_pack_time()
root._p_deactivate()
cn.sync()
self.assertTrue(listeq(root.keys(), ["key0", "key2"]))
L = db.undoInfo()
db.undo(L[0]["id"])
txn = transaction.get()
txn.note(u"undo deletion")
txn.commit()
set_pack_time()
root._p_deactivate()
cn.sync()
self.assertTrue(listeq(root.keys(), ["key0", "key1", "key2"]))
for t in pack_times:
self._storage.pack(t, referencesf)
root._p_deactivate()
cn.sync()
self.assertTrue(listeq(root.keys(), ["key0", "key1", "key2"]))
for i in range(3):
obj = root["key%d" % i]
self.assertEqual(obj.value, i)
root.items()
self._inter_pack_pause()
def checkPackAfterUndoManyTimes(self):
db = DB(self._storage)
cn = db.open()
try:
rt = cn.root()
rt["test"] = MinPO(1)
transaction.commit()
rt["test2"] = MinPO(2)
transaction.commit()
rt["test"] = MinPO(3)
txn = transaction.get()
txn.note(u"root of undo")
txn.commit()
packtimes = []
for i in range(10):
L = db.undoInfo()
db.undo(L[0]["id"])
txn = transaction.get()
txn.note(u"undo %d" % i)
txn.commit()
rt._p_deactivate()
cn.sync()
self.assertEqual(rt["test"].value, i % 2 and 3 or 1)
self.assertEqual(rt["test2"].value, 2)
packtimes.append(time.time())
snooze()
for t in packtimes:
self._storage.pack(t, referencesf)
cn.sync()
# TODO: Is _cache supposed to have a clear() method, or not?
# cn._cache.clear()
# The last undo set the value to 3 and pack should
# never change that.
self.assertEqual(rt["test"].value, 3)
self.assertEqual(rt["test2"].value, 2)
self._inter_pack_pause()
finally:
cn.close()
db.close()
def checkPackVersionsInPast(self):
db = DB(self._storage)
cn = db.open(version="testversion")
root = cn.root()
obj = root["obj"] = MinPO("obj")
root["obj2"] = MinPO("obj2")
txn = transaction.get()
txn.note("create 2 objs in version")
txn.commit()
obj.value = "77"
txn = transaction.get()
txn.note("modify obj in version")
txn.commit()
t0 = time.time()
snooze()
# undo the modification to generate a mix of backpointers
# and versions for pack to chase
info = db.undoInfo()
db.undo(info[0]["id"])
txn = transaction.get()
txn.note("undo modification")
txn.commit()
self._storage.pack(t0, referencesf)
db.commitVersion("testversion")
txn = transaction.get()
txn.note("commit version")
txn.commit()
cn = db.open()
root = cn.root()
root["obj"] = "no version"
txn = transaction.get()
txn.note("modify obj")
txn.commit()
self._storage.pack(time.time(), referencesf)
def checkPackAfterUndoManyTimes(self):
db = DB(self._storage)
cn = db.open()
rt = cn.root()
rt["test"] = MinPO(1)
transaction.commit()
rt["test2"] = MinPO(2)
transaction.commit()
rt["test"] = MinPO(3)
txn = transaction.get()
txn.note(u"root of undo")
txn.commit()
packtimes = []
for i in range(10):
L = db.undoInfo()
db.undo(L[0]["id"])
txn = transaction.get()
txn.note(u"undo %d" % i)
txn.commit()
rt._p_deactivate()
cn.sync()
self.assertEqual(rt["test"].value, i % 2 and 3 or 1)
self.assertEqual(rt["test2"].value, 2)
packtimes.append(time.time())
snooze()
for t in packtimes:
self._storage.pack(t, referencesf)
cn.sync()
# TODO: Is _cache supposed to have a clear() method, or not?
# cn._cache.clear()
# The last undo set the value to 3 and pack should
# never change that.
self.assertEqual(rt["test"].value, 3)
self.assertEqual(rt["test2"].value, 2)
self._inter_pack_pause()
def checkPackUnlinkedFromRoot(self):
eq = self.assertEqual
db = DB(self._storage)
conn = db.open()
root = conn.root()
txn = transaction.get()
txn.note('root')
txn.commit()
now = packtime = time.time()
while packtime <= now:
packtime = time.time()
obj = C()
obj.value = 7
root['obj'] = obj
txn = transaction.get()
txn.note('root -> o1')
txn.commit()
del root['obj']
txn = transaction.get()
txn.note('root -x-> o1')
txn.commit()
self._storage.pack(packtime, referencesf)
log = self._storage.undoLog()
tid = log[0]['id']
db.undo(tid)
txn = transaction.get()
txn.note('undo root -x-> o1')
txn.commit()
conn.sync()
eq(root['obj'].value, 7)
def checkPackUnlinkedFromRoot(self):
eq = self.assertEqual
db = DB(self._storage)
conn = db.open()
root = conn.root()
txn = transaction.get()
txn.note(u'root')
txn.commit()
now = packtime = time.time()
while packtime <= now:
packtime = time.time()
obj = C()
obj.value = 7
root['obj'] = obj
txn = transaction.get()
txn.note(u'root -> o1')
txn.commit()
del root['obj']
txn = transaction.get()
txn.note(u'root -x-> o1')
txn.commit()
self._storage.pack(packtime, referencesf)
log = self._storage.undoLog()
tid = log[0]['id']
db.undo(tid)
txn = transaction.get()
txn.note(u'undo root -x-> o1')
txn.commit()
conn.sync()
eq(root['obj'].value, 7)
def _gen_testdb(outfs_path, zext):
xtime_reset()
ext = ext4subj
if not zext:
def ext(subj): return {}
logging.basicConfig()
# generate random changes to objects hooked to top-level root by a/b/c/... key
random.seed(0)
namev = [_ for _ in "abcdefg"]
Niter = 2
for i in range(Niter):
stor = FileStorage(outfs_path, create=(i == 0))
db = DB(stor)
conn = db.open()
root = conn.root()
assert root._p_oid == p64(0), repr(root._p_oid)
for j in range(25):
name = random.choice(namev)
if name in root:
obj = root[name]
else:
root[name] = obj = Object(None)
obj.value = "%s%i.%i" % (name, i, j)
commit(u"user%i.%i" % (i,j), u"step %i.%i" % (i, j), ext(name))
# undo a transaction one step before a latest one a couple of times
for j in range(2):
# XXX undoLog, despite what its interface says:
# https://github.com/zopefoundation/ZODB/blob/2490ae09/src/ZODB/interfaces.py#L472
# just returns log of all transactions in specified range:
# https://github.com/zopefoundation/ZODB/blob/2490ae09/src/ZODB/FileStorage/FileStorage.py#L1008
# https://github.com/zopefoundation/ZODB/blob/2490ae09/src/ZODB/FileStorage/FileStorage.py#L2103
# so we retry undoing next log's txn on conflict.
for ul in db.undoLog(1, 20):
try:
db.undo(ul["id"])
commit(u"root%i.%i\nYour\nMagesty " % (i, j),
u"undo %i.%i\nmore detailed description\n\nzzz ..." % (i, j) + "\t"*(i+j),
ext("undo %s" % ul["id"]))
except UndoError:
transaction.abort()
continue
break
# delete an object
name = random.choice(list(root.keys()))
obj = root[name]
root[name] = Object("%s%i*" % (name, i))
# NOTE user/ext are kept empty on purpose - to also test this case
commit(u"", u"predelete %s" % unpack64(obj._p_oid), {})
# XXX obj in db could be changed by above undo, but ZODB does not automatically
# propagate undo changes to live objects - so obj._p_serial can be stale.
# Get serial via history.
obj_tid_lastchange = db.history(obj._p_oid)[0]['tid']
txn = precommit(u"root%i\nYour\nRoyal\nMagesty' " % i +
''.join(chr(_) for _ in range(32)), # <- NOTE all control characters
u"delete %i\nalpha beta gamma'delta\"lambda\n\nqqq ..." % i,
ext("delete %s" % unpack64(obj._p_oid)))
# at low level stor requires ZODB.IStorageTransactionMetaData not txn (ITransaction)
txn_stormeta = TransactionMetaData(txn.user, txn.description, txn.extension)
stor.tpc_begin(txn_stormeta)
stor.deleteObject(obj._p_oid, obj_tid_lastchange, txn_stormeta)
stor.tpc_vote(txn_stormeta)
# TODO different txn status vvv
# XXX vvv it does the thing, but py fs iterator treats this txn as EOF
#if i != Niter-1:
# stor.tpc_finish(txn_stormeta)
stor.tpc_finish(txn_stormeta)
# close db & rest not to get conflict errors after we touched stor
# directly a bit. everything will be reopened on next iteration.
conn.close()
db.close()
stor.close()
class Db_zodb:
def __init__(self, tkroot=None):
self.pathdb = None
self.pathdbrep = None
self.db_tuple = None
self.tkroot = tkroot
self.Listes = class_listes.Listes()
self.prefs = preferences.Preferences()
self.storage = None
self.db = None
self.connection = None
self.root = None
self.changed = None
self.noundo = 0
self.UNDO = False
def initialize(self, path):
self.pathdb = path
self.pathdbrep = os.path.normpath(os.path.split(os.path.abspath(path))[0])
class_tree.PATHDBREP = self.pathdbrep
self.clean()
# Connect to DB
self.storage = FileStorage.FileStorage(path)
self.db = DB(self.storage)
self.connection = self.db.open()
self.root = self.connection.root()
self.changed = False
def open(self, path=None):
"Open ZODB."
"""
Returns a db_tupe tuple consisting of:(root, connection, db, storage)
The same tuple must be passed to close_zodb() in order to close the DB.
"""
if path == None:
mess = 'please start by opening a database file'
if self.tkroot:
self.tkroot.dia.Message(mess)
else: print mess
return
if self.pathdb is not None:
self.save()
self.close()
self.pathdb = path
self.dbname = os.path.splitext(os.path.split(os.path.normpath(path))[1])[0]
self.initialize(path)
# get preferences
if 'listes' in self.root.keys():
self.Listes = self.root['listes']
self.Listes.update()
else:
self.root['listes'] = self.Listes
# get preferences
if 'preferences' in self.root.keys():
self.prefs = self.root['preferences']
self.prefs.checkprefs()
else:
self.root['preferences'] = self.prefs
if 'root_reps' in self.root.keys():
try:
self.root['root_reps'].reps
except:
# save to xml then convert
# convert to new style
rootreps = class_tree.Rootreps()
oldrootreps = self.root['root_reps']
for rootrep in self.root['root_reps'].values():
rootreps.add(rootrep)
rootreps.p_changed = 1
self.root['root_reps'] = rootreps
if 'root_reps' not in self.root:
self.root['root_reps'] = class_tree.Rootreps()
# say db is opened
mess = self.pathdb + ' database opened.'
self.refresh_gui()
self.db_tuple = (self.root, self.connection, self.db, self.storage)
self.make_backup()
def make_backup(self):
platforms.copy_file(self.pathdb, self.pathdb+'_backup')
def refresh_gui(self):
# apply esono changes in case gui is launched
if self.tkroot:
# refresh title and field names
self.tkroot.title('e-sonoclaste version 0.'+self.tkroot.version+' :: '+self.pathdb)
self.tkroot.editor.fields_buttons.change_fields_labels()
self.tkroot.editor.fields_buttons.change_fields_states()
# listes menu
# refreshes database
def sort_by_refs(self, rep):
rep.sort_by_refs()
for rep in rep.reps:
self.sort_by_refs(rep)
def refresh(self, tkroot=None):
for root_rep in self.root['root_reps'].reps:
tree_parse.parse(root_rep.path(), root_rep, refresh=True, tkroot=tkroot, db=self)
self.sort_by_refs(root_rep)
def add_rep(self, root_rep):
self.root['root_reps'].add(root_rep)
self.root['root_reps']._p_changed = 1
# print 'add_rep and save'
#self.save(gui=False)
def remove_rep(self, root_rep):
self.root['root_reps'].rm(root_rep)
self.root['root_reps']._p_changed = 1
def change_root(self, rootrep_path):
self.root['root_reps'].reps[0].fields.setref(rootrep_path)
#for root_rep in self.root['root_reps'].reps:
# root_rep.fields.setref(rootrep)
def clean(self):
# remove all database temporary files
try:
tmpfiles = glob.glob(self.pathdb+'.*')
for tmpfile in tmpfiles:
platforms.remove_file(tmpfile)
except:
print 'could not clean temporary files for database'
def close(self):
"Closes the ZODB."
"""
This function MUST be called at the end of each program !!!
"""
try:
self.pack()
transaction.abort()
self.db.close()
self.connection.close()
self.storage.close()
self.clean()
except:
print 'could not close database'
self.pathdb = None
self.db_tuple = None
self.storage = None
self.db = None
self.connection = None
self.root = None
self.changed = None
def save(self, gui=True):
if self.pathdb == None:
mess = 'please start by opening a database file'
if self.tkroot and gui:
self.tkroot.dia.message(mess)
else: print mess
return
transaction.commit()
self.changed = False
if self.tkroot and gui:
mess = self.tkroot.trad.saved()
# self.tkroot.dia.message(mess)
# print 'database saved'
else:
# print 'database saved'
pass
def auto_save(self):
transaction.commit()
if self.UNDO:
self.pack()
self.UNDO = False
self.noundo = 0
def undo_list(self, data_base):
"List of undo log, with latest transaction first."
"""
The time is converted into a readable time.
"""
undolog = data_base.undoLog(0, sys.maxint)
# convert the time stamp into something readable
for transact in undolog:
transact['time'] = time.ctime(transact['time'])
# convert into a list of lists
ret = []
for transact in undolog:
id = transact['id']
usr = transact['user_name']
tme = transact['time']
des = transact['description']
ret.append([tme,id,usr,des])
return ret
def undo(self):
undo_list = self.undo_list(self.db)
if not undo_list: return
if self.noundo>=len(undo_list): return
idundo = undo_list[self.noundo][1]
self.db.undo(idundo)
transaction.commit()
self.noundo+=2
self.tkroot.tree_gui.display_all()
self.tkroot.listes_gui.display()
undo_list = self.undo_list(self.db)
self.UNDO = True
def pack(self):
if self.db:
self.db.pack()
# =================================== SEARCH FUNCTIONS
def search(self, key, elt=None, fields_numbers=None):
rsl ={}
if not elt:
for root_rep in self.root['root_reps']:
rsl = self.search_rec(key, root_rep, fields_numbers, rsl)
else:
rsl = self.search_rec(key, elt, fields_numbers, rsl)
return rsl
def key2rsl(self, rsl, elt, fieldname):
if elt in rsl.keys():
rsl[elt].append(str(fieldname))
else:
rsl[elt] = [str(fieldname)]
return rsl
def search_in(self, key, elt, fields_numbers, rsl):
if elt.fields.name.lower().find(key.lower()) > -1:
rsl = self.key2rsl(rsl, elt, 'name')
if fields_numbers == None:
fields_numbers = range(len(elt.fields.contents))
for fieldno in fields_numbers:
content = elt.fields.contents[fieldno]
if content<>u'':
if content.lower().find(key.lower()) > -1:
rsl = self.key2rsl(rsl, elt, fieldno)
return rsl
def search_rec(self, key, elt, fields_numbers, rsl):
rsl = self.search_in(key, elt, fields_numbers, rsl)
if elt.type=='rep':
for f in elt.files:
rsl = self.search_rec(key, f, fields_numbers, rsl)
for marker in f.markers:
rsl = self.search_rec(key, marker, fields_numbers, rsl)
for rep in elt.reps:
rsl = self.search_rec(key, rep, fields_numbers, rsl)
return rsl
