def checkPackVersionReachable(self): db = DB(self._storage) cn = db.open() root = cn.root() names = "a", "b", "c" for name in names: root[name] = MinPO(name) transaction.commit() for name in names: cn2 = db.open(version=name) rt2 = cn2.root() obj = rt2[name] obj.value = MinPO("version") transaction.commit() cn2.close() root["d"] = MinPO("d") transaction.commit() snooze() self._storage.pack(time.time(), referencesf) cn.sync() # make sure all the non-version data is there for name, obj in root.items(): self.assertEqual(name, obj.value) # make sure all the version-data is there, # and create a new revision in the version for name in names: cn2 = db.open(version=name) rt2 = cn2.root() obj = rt2[name].value self.assertEqual(obj.value, "version") obj.value = "still version" transaction.commit() cn2.close() db.abortVersion("b") txn = transaction.get() txn.note("abort version b") txn.commit() t = time.time() snooze() L = db.undoInfo() db.undo(L[0]["id"]) txn = transaction.get() txn.note("undo abort") txn.commit() self._storage.pack(t, referencesf) cn2 = db.open(version="b") rt2 = cn2.root() self.assertEqual(rt2["b"].value.value, "still version")
def checkLoadBefore(self): # Store 10 revisions of one object and then make sure that we # can get all the non-current revisions back. oid = self._storage.new_oid() revs = [] revid = None for i in range(10): # We need to ensure that successive timestamps are at least # two apart, so that a timestamp exists that's unambiguously # between successive timestamps. Each call to snooze() # guarantees that the next timestamp will be at least one # larger (and probably much more than that) than the previous # one. snooze() snooze() revid = self._dostore(oid, revid, data=MinPO(i)) revs.append(self._storage.loadEx(oid, "")) prev = u64(revs[0][1]) for i in range(1, 10): tid = revs[i][1] cur = u64(tid) middle = prev + (cur - prev) // 2 assert prev < middle < cur # else the snooze() trick failed prev = cur t = self._storage.loadBefore(oid, p64(middle)) self.assert_(t is not None) data, start, end = t self.assertEqual(revs[i - 1][0], data) self.assertEqual(tid, end)
def checkPackLotsWhileWriting(self): # This is like the other pack-while-writing tests, except it packs # repeatedly until the client thread is done. At the time it was # introduced, it reliably provoked # CorruptedError: ... transaction with checkpoint flag set # in the ZEO flavor of the FileStorage tests. db = DB(self._storage) conn = db.open() root = conn.root() choices = range(10) for i in choices: root[i] = MinPO(i) transaction.commit() snooze() packt = time.time() for dummy in choices: for i in choices: root[i].value = MinPO(i) transaction.commit() NUM_LOOP_TRIP = 100 timer = ElapsedTimer(time.time()) thread = ClientThread(db, choices, NUM_LOOP_TRIP, timer, 0) thread.start() while thread.isAlive(): db.pack(packt) snooze() packt = time.time() thread.join() self._sanity_check()
def checkLoadBefore(self): # Store 10 revisions of one object and then make sure that we # can get all the non-current revisions back. oid = self._storage.new_oid() revs = [] revid = None for i in range(10): # We need to ensure that successive timestamps are at least # two apart, so that a timestamp exists that's unambiguously # between successive timestamps. Each call to snooze() # guarantees that the next timestamp will be at least one # larger (and probably much more than that) than the previous # one. snooze() snooze() revid = self._dostore(oid, revid, data=MinPO(i)) revs.append(load_current(self._storage, oid)) prev = u64(revs[0][1]) for i in range(1, 10): tid = revs[i][1] cur = u64(tid) middle = prev + (cur - prev) // 2 assert prev < middle < cur # else the snooze() trick failed prev = cur t = self._storage.loadBefore(oid, p64(middle)) self.assertTrue(t is not None) data, start, end = t self.assertEqual(revs[i-1][0], data) self.assertEqual(tid, end)
def checkPackWithGCOnDestinationAfterRestore(self): raises = self.assertRaises db = DB(self._storage) conn = db.open() root = conn.root() root.obj = obj1 = MinPO(1) txn = transaction.get() txn.note('root -> obj') txn.commit() root.obj.obj = obj2 = MinPO(2) txn = transaction.get() txn.note('root -> obj -> obj') txn.commit() del root.obj txn = transaction.get() txn.note('root -X->') txn.commit() # Now copy the transactions to the destination self._dst.copyTransactionsFrom(self._storage) # Now pack the destination. snooze() self._dst.pack(time.time(), referencesf) # And check to see that the root object exists, but not the other # objects. data, serial = self._dst.load(root._p_oid, '') raises(KeyError, self._dst.load, obj1._p_oid, '') raises(KeyError, self._dst.load, obj2._p_oid, '')
def checkPackLotsWhileWriting(self): # This is like the other pack-while-writing tests, except it packs # repeatedly until the client thread is done. At the time it was # introduced, it reliably provoked # CorruptedError: ... transaction with checkpoint flag set # in the ZEO flavor of the FileStorage tests. db = DB(self._storage) conn = db.open() root = conn.root() choices = list(range(10)) for i in choices: root[i] = MinPO(i) transaction.commit() snooze() packt = time.time() for dummy in choices: for i in choices: root[i].value = MinPO(i) transaction.commit() NUM_LOOP_TRIP = 100 timer = ElapsedTimer(time.time()) thread = ClientThread(db, choices, NUM_LOOP_TRIP, timer, 0) thread.start() while thread.isAlive(): db.pack(packt) snooze() packt = time.time() thread.join() self._sanity_check()
def checkPackWithGCOnDestinationAfterRestore(self): raises = self.assertRaises db = DB(self._storage) conn = db.open() root = conn.root() root.obj = obj1 = MinPO(1) txn = transaction.get() txn.note("root -> obj") txn.commit() root.obj.obj = obj2 = MinPO(2) txn = transaction.get() txn.note("root -> obj -> obj") txn.commit() del root.obj txn = transaction.get() txn.note("root -X->") txn.commit() # Now copy the transactions to the destination self._dst.copyTransactionsFrom(self._storage) # Now pack the destination. snooze() self._dst.pack(time.time(), referencesf) # And check to see that the root object exists, but not the other # objects. data, serial = self._dst.load(root._p_oid, "") raises(KeyError, self._dst.load, obj1._p_oid, "") raises(KeyError, self._dst.load, obj2._p_oid, "")
def dont_checkPackUndoLogUndoable(self): # A disabled test. I wanted to test that the content of the # undo log was consistent, but every storage appears to # include something slightly different. If the result of this # method is only used to fill a GUI then this difference # doesnt matter. Perhaps re-enable this test once we agree # what should be asserted. self._initroot() # Create two `persistent' object obj1 = self._newobj() oid1 = obj1.getoid() obj1.value = 1 obj2 = self._newobj() oid2 = obj2.getoid() obj2.value = 2 # Commit the first revision of each of them revid11 = self._dostoreNP(oid1, data=pdumps(obj1), description="1-1") revid22 = self._dostoreNP(oid2, data=pdumps(obj2), description="2-2") # remember the time. everything above here will be packed away snooze() packtime = time.time() snooze() # Commit two revisions of the first object obj1.value = 3 revid13 = self._dostoreNP(oid1, revid=revid11, data=pdumps(obj1), description="1-3") obj1.value = 4 self._dostoreNP(oid1, revid=revid13, data=pdumps(obj1), description="1-4") # Commit one revision of the second object obj2.value = 5 self._dostoreNP(oid2, revid=revid22, data=pdumps(obj2), description="2-5") # Now pack self.assertEqual(6, len(self._storage.undoLog())) print('\ninitial undoLog was') for r in self._storage.undoLog(): print(r) self._storage.pack(packtime, referencesf) # The undo log contains only two undoable transaction. print('\nafter packing undoLog was') for r in self._storage.undoLog(): print(r)
def dont_checkPackUndoLogUndoable(self): # A disabled test. I wanted to test that the content of the # undo log was consistent, but every storage appears to # include something slightly different. If the result of this # method is only used to fill a GUI then this difference # doesnt matter. Perhaps re-enable this test once we agree # what should be asserted. self._initroot() # Create two `persistent' object obj1 = self._newobj() oid1 = obj1.getoid() obj1.value = 1 obj2 = self._newobj() oid2 = obj2.getoid() obj2.value = 2 # Commit the first revision of each of them revid11 = self._dostoreNP(oid1, data=pdumps(obj1), description="1-1") revid22 = self._dostoreNP(oid2, data=pdumps(obj2), description="2-2") # remember the time. everything above here will be packed away snooze() packtime = time.time() snooze() # Commit two revisions of the first object obj1.value = 3 revid13 = self._dostoreNP(oid1, revid=revid11, data=pdumps(obj1), description="1-3") obj1.value = 4 self._dostoreNP(oid1, revid=revid13, data=pdumps(obj1), description="1-4") # Commit one revision of the second object obj2.value = 5 self._dostoreNP(oid2, revid=revid22, data=pdumps(obj2), description="2-5") # Now pack self.assertEqual(6,len(self._storage.undoLog())) print('\ninitial undoLog was') for r in self._storage.undoLog(): print(r) self._storage.pack(packtime, referencesf) # The undo log contains only two undoable transaction. print('\nafter packing undoLog was') for r in self._storage.undoLog(): print(r)
def checkPackUndoLog(self): self._initroot() # Create a `persistent' object obj = self._newobj() oid = obj.getoid() obj.value = 1 # Commit two different revisions revid1 = self._dostoreNP(oid, data=pdumps(obj)) obj.value = 2 snooze() packtime = time.time() snooze() self._dostoreNP(oid, revid=revid1, data=pdumps(obj)) # Now pack the first transaction self.assertEqual(3, len(self._storage.undoLog())) self._storage.pack(packtime, referencesf) # The undo log contains only the most resent transaction self.assertEqual(1, len(self._storage.undoLog()))
def checkPackVersionsInPast(self): db = DB(self._storage) cn = db.open(version="testversion") root = cn.root() obj = root["obj"] = MinPO("obj") root["obj2"] = MinPO("obj2") txn = transaction.get() txn.note("create 2 objs in version") txn.commit() obj.value = "77" txn = transaction.get() txn.note("modify obj in version") txn.commit() t0 = time.time() snooze() # undo the modification to generate a mix of backpointers # and versions for pack to chase info = db.undoInfo() db.undo(info[0]["id"]) txn = transaction.get() txn.note("undo modification") txn.commit() self._storage.pack(t0, referencesf) db.commitVersion("testversion") txn = transaction.get() txn.note("commit version") txn.commit() cn = db.open() root = cn.root() root["obj"] = "no version" txn = transaction.get() txn.note("modify obj") txn.commit() self._storage.pack(time.time(), referencesf)
def checkPackLotsWhileWriting(self): # This is like the other pack-while-writing tests, except it packs # repeatedly until the client thread is done. At the time it was # introduced, it reliably provoked # CorruptedError: ... transaction with checkpoint flag set # in the ZEO flavor of the FileStorage tests. db = DB(self._storage) conn = db.open() root = conn.root() choices = range(10) for i in choices: root[i] = MinPO(i) transaction.commit() snooze() packt = time.time() for dummy in choices: for i in choices: root[i].value = MinPO(i) transaction.commit() NUM_LOOP_TRIP = 100 timer = ElapsedTimer(time.time()) thread = ClientThread(db, choices, NUM_LOOP_TRIP, timer, 0) thread.start() while thread.isAlive(): db.pack(packt) snooze() packt = time.time() thread.join() # Iterate over the storage to make sure it's sane. if not hasattr(self._storage, "iterator"): return it = self._storage.iterator() for txn in it: for data in txn: pass it.close()
def checkPackWithGCOnDestinationAfterRestore(self): raises = self.assertRaises closing = self._closing db = closing(DB(self._storage)) conn = closing(db.open()) root = conn.root() root.obj = obj1 = MinPO(1) txn = transaction.get() txn.note(u'root -> obj') txn.commit() root.obj.obj = obj2 = MinPO(2) txn = transaction.get() txn.note(u'root -> obj -> obj') txn.commit() del root.obj txn = transaction.get() txn.note(u'root -X->') txn.commit() # Now copy the transactions to the destination self._dst.copyTransactionsFrom(self._storage) # If the source storage is a history-free storage, all # of the transactions are now marked as packed in the # destination storage. To trigger a pack, we have to # add another transaction to the destination that is # not packed. db2 = closing(DB(self._dst)) conn2 = closing(db2.open()) conn2.root().extra = 0 txn = transaction.get() txn.note(u'root.extra = 0') txn.commit() # Now pack the destination. snooze() self._dst.pack(time.time(), referencesf) # And check to see that the root object exists, but not the other # objects. _data, _serial = self._dst.load(root._p_oid, '') raises(KeyError, self._dst.load, obj1._p_oid, '') raises(KeyError, self._dst.load, obj2._p_oid, '')
def checkRedundantPack(self): # It is an error to perform a pack with a packtime earlier # than a previous packtime. The storage can't do a full # traversal as of the packtime, because the previous pack may # have removed revisions necessary for a full traversal. # It should be simple to test that a storage error is raised, # but this test case goes to the trouble of constructing a # scenario that would lose data if the earlier packtime was # honored. self._initroot() db = DB(self._storage) conn = db.open() root = conn.root() root["d"] = d = PersistentMapping() transaction.commit() snooze() obj = d["obj"] = C() obj.value = 1 transaction.commit() snooze() packt1 = time.time() lost_oid = obj._p_oid obj = d["anotherobj"] = C() obj.value = 2 transaction.commit() snooze() packt2 = time.time() db.pack(packt2) # BDBStorage allows the second pack, but doesn't lose data. try: db.pack(packt1) except StorageError: pass # This object would be removed by the second pack, even though # it is reachable. load_current(self._storage, lost_oid)
def checkRedundantPack(self): # It is an error to perform a pack with a packtime earlier # than a previous packtime. The storage can't do a full # traversal as of the packtime, because the previous pack may # have removed revisions necessary for a full traversal. # It should be simple to test that a storage error is raised, # but this test case goes to the trouble of constructing a # scenario that would lose data if the earlier packtime was # honored. self._initroot() db = DB(self._storage) conn = db.open() root = conn.root() root["d"] = d = PersistentMapping() transaction.commit() snooze() obj = d["obj"] = C() obj.value = 1 transaction.commit() snooze() packt1 = time.time() lost_oid = obj._p_oid obj = d["anotherobj"] = C() obj.value = 2 transaction.commit() snooze() packt2 = time.time() db.pack(packt2) # BDBStorage allows the second pack, but doesn't lose data. try: db.pack(packt1) except StorageError: pass # This object would be removed by the second pack, even though # it is reachable. self._storage.load(lost_oid, "")
def _PackWhileWriting(self, pack_now): # A storage should allow some reading and writing during # a pack. This test attempts to exercise locking code # in the storage to test that it is safe. It generates # a lot of revisions, so that pack takes a long time. db = DB(self._storage) conn = db.open() root = conn.root() for i in range(10): root[i] = MinPO(i) transaction.commit() snooze() packt = time.time() choices = list(range(10)) for dummy in choices: for i in choices: root[i].value = MinPO(i) transaction.commit() # How many client threads should we run, and how long should we # wait for them to finish? Hard to say. Running 4 threads and # waiting 30 seconds too often left a thread still alive on Tim's # Win98SE box, during ZEO flavors of this test. Those tend to # run one thread at a time to completion, and take about 10 seconds # per thread. There doesn't appear to be a compelling reason to # run that many threads. Running 3 threads and waiting up to a # minute seems to work well in practice. The ZEO tests normally # finish faster than that, and the non-ZEO tests very much faster # than that. NUM_LOOP_TRIP = 50 timer = ElapsedTimer(time.time()) threads = [ClientThread(db, choices, NUM_LOOP_TRIP, timer, i) for i in range(3)] for t in threads: t.start() if pack_now: db.pack(time.time()) else: db.pack(packt) for t in threads: t.join(60) liveness = [t.isAlive() for t in threads] if True in liveness: # They should have finished by now. print('Liveness:', liveness) # Combine the outcomes, and sort by start time. outcomes = [] for t in threads: outcomes.extend(t.outcomes) # each outcome list has as many of these as a loop trip got thru: # thread_id # elapsed millis at loop top # elapsed millis at attempt to assign to self.root[index] # index into self.root getting replaced # elapsed millis when outcome known # 'OK' or 'Conflict' # True if we got beyond this line, False if it raised an # exception (one possible Conflict cause): # self.root[index].value = MinPO(j) def cmp_by_time(a, b): return cmp((a[1], a[0]), (b[1], b[0])) outcomes.sort(cmp_by_time) counts = [0] * 4 for outcome in outcomes: n = len(outcome) assert n >= 2 tid = outcome[0] print('tid:%d top:%5d' % (tid, outcome[1]), end=' ') if n > 2: print('commit:%5d' % outcome[2], end=' ') if n > 3: print('index:%2d' % outcome[3], end=' ') if n > 4: print('known:%5d' % outcome[4], end=' ') if n > 5: print('%8s' % outcome[5], end=' ') if n > 6: print('assigned:%5s' % outcome[6], end=' ') counts[tid] += 1 if counts[tid] == NUM_LOOP_TRIP: print('thread %d done' % tid, end=' ') print() self.fail('a thread is still alive') self._sanity_check() db.close()
def _PackWhileWriting(self, pack_now): # A storage should allow some reading and writing during # a pack. This test attempts to exercise locking code # in the storage to test that it is safe. It generates # a lot of revisions, so that pack takes a long time. db = DB(self._storage) conn = db.open() root = conn.root() for i in range(10): root[i] = MinPO(i) transaction.commit() snooze() packt = time.time() choices = list(range(10)) for dummy in choices: for i in choices: root[i].value = MinPO(i) transaction.commit() # How many client threads should we run, and how long should we # wait for them to finish? Hard to say. Running 4 threads and # waiting 30 seconds too often left a thread still alive on Tim's # Win98SE box, during ZEO flavors of this test. Those tend to # run one thread at a time to completion, and take about 10 seconds # per thread. There doesn't appear to be a compelling reason to # run that many threads. Running 3 threads and waiting up to a # minute seems to work well in practice. The ZEO tests normally # finish faster than that, and the non-ZEO tests very much faster # than that. NUM_LOOP_TRIP = 50 timer = ElapsedTimer(time.time()) threads = [ ClientThread(db, choices, NUM_LOOP_TRIP, timer, i) for i in range(3) ] for t in threads: t.start() if pack_now: db.pack(time.time()) else: db.pack(packt) for t in threads: t.join(60) liveness = [t.isAlive() for t in threads] if True in liveness: # They should have finished by now. print('Liveness:', liveness) # Combine the outcomes, and sort by start time. outcomes = [] for t in threads: outcomes.extend(t.outcomes) # each outcome list has as many of these as a loop trip got thru: # thread_id # elapsed millis at loop top # elapsed millis at attempt to assign to self.root[index] # index into self.root getting replaced # elapsed millis when outcome known # 'OK' or 'Conflict' # True if we got beyond this line, False if it raised an # exception (one possible Conflict cause): # self.root[index].value = MinPO(j) def cmp_by_time(a, b): return cmp((a[1], a[0]), (b[1], b[0])) outcomes.sort(cmp_by_time) counts = [0] * 4 for outcome in outcomes: n = len(outcome) assert n >= 2 tid = outcome[0] print('tid:%d top:%5d' % (tid, outcome[1]), end=' ') if n > 2: print('commit:%5d' % outcome[2], end=' ') if n > 3: print('index:%2d' % outcome[3], end=' ') if n > 4: print('known:%5d' % outcome[4], end=' ') if n > 5: print('%8s' % outcome[5], end=' ') if n > 6: print('assigned:%5s' % outcome[6], end=' ') counts[tid] += 1 if counts[tid] == NUM_LOOP_TRIP: print('thread %d done' % tid, end=' ') print() self.fail('a thread is still alive') self._sanity_check()