def purge(self, sets): topurge = rangeset.RangeSet() committed = sets[COMMITTED] l = committed.length() while l > self.max_journal_history: first = committed[0] # a range contains a single txid r = rangeset.RangeSet([[first[0], first[0] + 1]]) topurge.add(r[0]) committed = rangeset.substract(committed, r) l -= 1 for rng in topurge: for txid in range(rng[0], rng[1]): self.journal.safe_delete(txid) sets[PURGED] = rangeset.union(sets[PURGED], topurge) sets[COMMITTED] = rangeset.substract(sets[COMMITTED], topurge)
def test_concurrent_3_record(self): n_tx = 10 ks = ('foo', 'bar', 'duu') def _tx(i): while True: try: with ZKTransaction(zkhost) as t1: for ii in range(len(ks)): k = ks[(ii+i) % len(ks)] foo = t1.lock_get(k) foo.v = foo.v or 0 foo.v += 1 t1.set(foo) t1.commit() dd(str(t1) + ' committed') return except (Deadlock, HigherTXApplied) as e: dd(str(t1) + ': ' + repr(e)) continue except TXTimeout as e: dd(str(t1) + ': ' + repr(e)) raise with ututil.Timer() as tt: for th in [threadutil.start_daemon(_tx, args=(i, )) for i in range(n_tx)]: th.join() dd('3 key 10 thread: ', tt.spent()) t = ZKTransaction(zkhost) for k in ks: rst, ver = t.zkstorage.record.get(k) dd(rst) self.assertEqual(n_tx, rst[-1][1]) rst, ver = t.zkstorage.txidset.get() dd(rst) self.assertEqual(n_tx, rst[COMMITTED].length()) # check aborted txidset sm = rangeset.union(rst[COMMITTED], rst[PURGED]) self.assertEqual(rst[COMMITTED][-1][-1] - 1, sm.length())
def redo_all_dead_tx(self): sets, ver = self.zkstorage.txidset.get() # Only check for holes in the txid range set. # With `txidset = [[1, 2], [4, 5]]`, tx-2 and tx-3 is unfinished tx. # Check these tx # A tx is committed or purged that does not need to handle it again. known = rangeset.union(sets[COMMITTED], sets[PURGED]) for rng in known[:-1]: txid = rng[1] if not self.is_tx_alive(txid) and not self.has_state(txid): self.redo_dead_tx(txid)
def test_int_inc_union(self): cases = ( ([[None, 10], [20, 30], [40, None]], [[None, None]], [[None, None]]), ([[None, 10], [20, 30], [40, None]], [[None, 1]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[None, 10]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[None, 11]], [[None, 11], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[9, 11]], [[None, 11], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[10, 11]], [[None, 11], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[11, 12]], [[None, 12], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[12, 13]], [[None, 10], [12, 13], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[18, 19]], [[None, 10], [18, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[19, 21]], [[None, 10], [19, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[20, 21]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[24, 25]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[29, 30]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[29, 31]], [[None, 10], [20, 31], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[30, 31]], [[None, 10], [20, 31], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[31, 32]], [[None, 10], [20, 32], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[32, 33]], [[None, 10], [20, 30], [32, 33], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[39, 39]], [[None, 10], [20, 30], [39, None]]), ([[None, 10], [20, 30], [40, None]], [[39, 40]], [[None, 10], [20, 30], [39, None]]), ([[None, 10], [20, 30], [40, None]], [[39, 41]], [[None, 10], [20, 30], [39, None]]), ([[None, 10], [20, 30], [40, None]], [[40, 41]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[41, 42]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[41, None]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[8, 25], [35, 40]], [[None, 30], [35, None]]), ) dd() for a, b, expected in cases: dd('case:', a, b, expected) a = rangeset.RangeSet(a, range_clz=rangeset.IntIncRange) b = rangeset.RangeSet(b, range_clz=rangeset.IntIncRange) rst = rangeset.union(a, b) dd('rst:', rst) self.assertEqual(expected, rst)
def purge(self, sets): committed_set = sets[COMMITTED] length = committed_set.length() if length <= self.max_journal_history: return purged_set = sets[PURGED] purged_end_jid = length - self.max_journal_history topurge = rangeset.RangeSet() for jid in range(purged_end_jid): if purged_set.has(jid): continue topurge.add([jid, jid + 1]) for rng in topurge: for jid in range(rng[0], rng[1]): self.journal.safe_delete('{jid:0>10}'.format(jid=jid)) sets[PURGED] = rangeset.union(sets[PURGED], topurge)