def purge(self, sets):
topurge = rangeset.RangeSet()
committed = sets[COMMITTED]
l = committed.length()
while l > self.max_journal_history:
first = committed[0]
# a range contains a single txid
r = rangeset.RangeSet([[first[0], first[0] + 1]])
topurge.add(r[0])
committed = rangeset.substract(committed, r)
l -= 1
for rng in topurge:
for txid in range(rng[0], rng[1]):
self.journal.safe_delete(txid)
sets[PURGED] = rangeset.union(sets[PURGED], topurge)
sets[COMMITTED] = rangeset.substract(sets[COMMITTED], topurge)
def test_concurrent_3_record(self):
n_tx = 10
ks = ('foo', 'bar', 'duu')
def _tx(i):
while True:
try:
with ZKTransaction(zkhost) as t1:
for ii in range(len(ks)):
k = ks[(ii+i) % len(ks)]
foo = t1.lock_get(k)
foo.v = foo.v or 0
foo.v += 1
t1.set(foo)
t1.commit()
dd(str(t1) + ' committed')
return
except (Deadlock,
HigherTXApplied) as e:
dd(str(t1) + ': ' + repr(e))
continue
except TXTimeout as e:
dd(str(t1) + ': ' + repr(e))
raise
with ututil.Timer() as tt:
for th in [threadutil.start_daemon(_tx, args=(i, ))
for i in range(n_tx)]:
th.join()
dd('3 key 10 thread: ', tt.spent())
t = ZKTransaction(zkhost)
for k in ks:
rst, ver = t.zkstorage.record.get(k)
dd(rst)
self.assertEqual(n_tx, rst[-1][1])
rst, ver = t.zkstorage.txidset.get()
dd(rst)
self.assertEqual(n_tx, rst[COMMITTED].length())
# check aborted txidset
sm = rangeset.union(rst[COMMITTED], rst[PURGED])
self.assertEqual(rst[COMMITTED][-1][-1] - 1, sm.length())
def redo_all_dead_tx(self):
sets, ver = self.zkstorage.txidset.get()
# Only check for holes in the txid range set.
# With `txidset = [[1, 2], [4, 5]]`, tx-2 and tx-3 is unfinished tx.
# Check these tx
# A tx is committed or purged that does not need to handle it again.
known = rangeset.union(sets[COMMITTED], sets[PURGED])
for rng in known[:-1]:
txid = rng[1]
if not self.is_tx_alive(txid) and not self.has_state(txid):
self.redo_dead_tx(txid)
def test_int_inc_union(self):
cases = (
([[None, 10], [20, 30], [40, None]], [[None, None]], [[None, None]]),
([[None, 10], [20, 30], [40, None]], [[None, 1]], [[None, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[None, 10]], [[None, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[None, 11]], [[None, 11], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[9, 11]], [[None, 11], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[10, 11]], [[None, 11], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[11, 12]], [[None, 12], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[12, 13]], [[None, 10], [12, 13], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[18, 19]], [[None, 10], [18, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[19, 21]], [[None, 10], [19, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[20, 21]], [[None, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[24, 25]], [[None, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[29, 30]], [[None, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[29, 31]], [[None, 10], [20, 31], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[30, 31]], [[None, 10], [20, 31], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[31, 32]], [[None, 10], [20, 32], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[32, 33]], [[None, 10], [20, 30], [32, 33], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[39, 39]], [[None, 10], [20, 30], [39, None]]),
([[None, 10], [20, 30], [40, None]], [[39, 40]], [[None, 10], [20, 30], [39, None]]),
([[None, 10], [20, 30], [40, None]], [[39, 41]], [[None, 10], [20, 30], [39, None]]),
([[None, 10], [20, 30], [40, None]], [[40, 41]], [[None, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[41, 42]], [[None, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[41, None]], [[None, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[8, 25], [35, 40]], [[None, 30], [35, None]]),
)
dd()
for a, b, expected in cases:
dd('case:', a, b, expected)
a = rangeset.RangeSet(a, range_clz=rangeset.IntIncRange)
b = rangeset.RangeSet(b, range_clz=rangeset.IntIncRange)
rst = rangeset.union(a, b)
dd('rst:', rst)
self.assertEqual(expected, rst)
def purge(self, sets):
committed_set = sets[COMMITTED]
length = committed_set.length()
if length <= self.max_journal_history:
return
purged_set = sets[PURGED]
purged_end_jid = length - self.max_journal_history
topurge = rangeset.RangeSet()
for jid in range(purged_end_jid):
if purged_set.has(jid):
continue
topurge.add([jid, jid + 1])
for rng in topurge:
for jid in range(rng[0], rng[1]):
self.journal.safe_delete('{jid:0>10}'.format(jid=jid))
sets[PURGED] = rangeset.union(sets[PURGED], topurge)