def purge(self, sets):
topurge = rangeset.RangeSet()
committed = sets[COMMITTED]
l = committed.length()
while l > self.max_journal_history:
first = committed[0]
# a range contains a single txid
r = rangeset.RangeSet([[first[0], first[0] + 1]])
topurge.add(r[0])
committed = rangeset.substract(committed, r)
l -= 1
for rng in topurge:
for txid in range(rng[0], rng[1]):
self.journal.safe_delete(txid)
sets[PURGED] = rangeset.union(sets[PURGED], topurge)
sets[COMMITTED] = rangeset.substract(sets[COMMITTED], topurge)
def compare(a, b):
rst = {
'onlyleft': {
'length': 0,
'gtidset': {},
},
'onlyright': {
'length': 0,
'gtidset': {},
},
}
for uuid, intervals in a.items():
ll = rangeset.substract(intervals,
b.get(uuid, rangeset.IntIncRangeSet([])))
if ll.length() > 0:
rst['onlyleft']['gtidset'][uuid] = ll
rst['onlyleft']['length'] += ll.length()
for uuid, intervals in b.items():
ll = rangeset.substract(intervals,
a.get(uuid, rangeset.IntIncRangeSet([])))
if ll.length() > 0:
rst['onlyright']['gtidset'][uuid] = ll
rst['onlyright']['length'] += ll.length()
return rst
def compare(a, b):
rst = {
'onlyleft': {
'length': 0,
'gtidset': {},
},
'onlyright': {
'length': 0,
'gtidset': {},
},
}
for uuid, intervals in a.items():
ll = rangeset.substract(intervals, b.get(uuid, rangeset.IntIncRangeSet([])))
if ll.length() > 0:
rst['onlyleft']['gtidset'][uuid] = ll
rst['onlyleft']['length'] += ll.length()
for uuid, intervals in b.items():
ll = rangeset.substract(intervals, a.get(uuid, rangeset.IntIncRangeSet([])))
if ll.length() > 0:
rst['onlyright']['gtidset'][uuid] = ll
rst['onlyright']['length'] += ll.length()
return rst
def test_substract(self):
cases = (
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[None, None]], []),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[None, 1]], [[1, 10, 'a'], [20, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[None, 10]], [ [20, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[None, 11]], [ [20, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[9, 11]], [[None, 9, 'a'], [20, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[10, 11]], [[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[11, 12]], [[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[19, 20]], [[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[19, 21]], [[None, 10, 'a'], [21, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[20, 21]], [[None, 10, 'a'], [21, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[24, 25]], [[None, 10, 'a'], [20, 24, 'b'], [25, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[29, 30]], [[None, 10, 'a'], [20, 29, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[29, 31]], [[None, 10, 'a'], [20, 29, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[30, 31]], [[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[31, 32]], [[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[39, 40]], [[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[39, 41]], [[None, 10, 'a'], [20, 30, 'b'], [41, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[40, 41]], [[None, 10, 'a'], [20, 30, 'b'], [41, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[41, 42]], [[None, 10, 'a'], [20, 30, 'b'], [40, 41, 'c'], [42, None, 'c']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[41, None]], [[None, 10, 'a'], [20, 30, 'b'], [40, 41, 'c'], ]),
([[20, 30, 'a'] ], [[20, 24], [25, 30]], [[24, 25, 'a']]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[20, 24], [25, 30]], [[None, 10, 'a'], [24, 25, 'b'], [40, None, 'c'], ]),
([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [[1, 2], [8, 25], [35, 45]], [[None, 1, 'a'], [2, 8, 'a'], [25, 30, 'b'], [45, None, 'c']]),
)
dd()
for a, b, expected in cases:
dd('case:', a, b, expected)
# RangeDict - RangeSet
a = rangeset.RangeDict(a)
b = rangeset.RangeSet(b)
rst = rangeset.substract(a, b)
dd('rst:', rst)
self.assertEqual(expected, rst)
# RangeDict - RangeDict
b = rangeset.RangeDict(b)
rst = rangeset.substract(a, b)
dd('rst:', rst)
self.assertEqual(expected, rst)
def test_substract(self):
r = rangeset.RangeDict(self.inp, dimension=2)
r = rangeset.substract(r, rangeset.RangeDict([[0.5, 1.5, None]]))
dd(r)
self.assertRaises(KeyError, r.get, 2)
self.assertRaises(KeyError, r.get, 2, 'e')
self.assertEqual('bd', r.get(0, 'b'))
self.assertEqual('ac', r.get(1.6, 'b'))
r = rangeset.substract(r, rangeset.RangeDict([[0, 1.5, None]]))
dd(r)
self.assertRaises(KeyError, r.get, 0)
def _get_uncommitted_txids(self):
storage_txidset = self.storage.txidset.get()
subset = rangeset.substract(self.zk_cmt_txidset,
storage_txidset[COMMITTED])
for begin, end in subset:
for i in range(begin, end):
yield i
def _get_uncommitted_journal_ids(self):
storage_journal_id_set = self.storage.journal_id_set.get()
subset = rangeset.substract(self.zk_journal_id_set[COMMITTED],
storage_journal_id_set[COMMITTED])
for begin, end in subset:
for jid in range(begin, end):
yield jid
def _get_uncommitted_journal_ids(self):
storage_journal_id_set = self.storage.journal_id_set.get()
subset = rangeset.substract(self.zk_journal_id_set[COMMITTED],
storage_journal_id_set[COMMITTED])
for begin, end in subset:
for jid in range(begin, end):
yield jid
def test_int_inc_substract(self):
cases = (
([[None, 10], [20, 30], [40, None]], [[None, None]], []),
([[None, 10], [20, 30], [40, None]], [[None, 1]], [[2, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[None, 10]], [[20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[None, 11]], [[20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[9, 11]], [[None, 8], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[10, 11]], [[None, 9], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[11, 12]], [[None, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[19, 20]], [[None, 10], [21, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[19, 21]], [[None, 10], [22, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[20, 21]], [[None, 10], [22, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[24, 25]], [[None, 10], [20, 23], [26, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[29, 30]], [[None, 10], [20, 28], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[29, 31]], [[None, 10], [20, 28], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[30, 31]], [[None, 10], [20, 29], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[31, 32]], [[None, 10], [20, 30], [40, None]]),
([[None, 10], [20, 30], [40, None]], [[39, 40]], [[None, 10], [20, 30], [41, None]]),
([[None, 10], [20, 30], [40, None]], [[39, 41]], [[None, 10], [20, 30], [42, None]]),
([[None, 10], [20, 30], [40, None]], [[40, 41]], [[None, 10], [20, 30], [42, None]]),
([[None, 10], [20, 30], [40, None]], [[41, 42]], [[None, 10], [20, 30], [40, 40], [43, None]]),
([[None, 10], [20, 30], [40, None]], [[41, None]], [[None, 10], [20, 30], [40, 40], ]),
([[20, 30]], [[20, 23], [25, 30]], [[24, 24]]),
([[20, 30]], [[20, 22], [27, 30]], [[23, 26]]),
([[None, 10], [20, 30], [40, None]], [[20, 23], [26, 30]], [[None, 10], [24, 25], [40, None], ]),
([[None, 10], [20, 30], [40, None]], [[1, 2], [8, 25], [35, 45]], [[None, 0], [3, 7], [26, 30], [46, None]]),
)
dd()
for a, b, expected in cases:
dd('case:', a, b, expected)
a = rangeset.RangeSet(a, range_clz=rangeset.IntIncRange)
b = rangeset.RangeSet(b, range_clz=rangeset.IntIncRange)
rst = rangeset.substract(a, b)
dd('rst:', rst)
self.assertEqual(expected, rst)