def test_apply(self): cases = ( { 'meta/server/k1': 10, 'meta/region/k2': 100 }, { 'meta/server/k3': 'server_v1', 'meta/region/k4': 'region_v2' }, { 'meta/server/k5': {}, 'meta/region/k6': {} }, { 'meta/server/k7': { 'x': 'x', 'y': 'y' }, 'meta/region/k8': { 'x': 'x', 'y': 'y' } }, { 'meta/server/k9': [], 'meta/region/k10': [] }, { 'meta/server/k11': [1, 2], 'meta/region/k12': [4, 5] }, ) max_txid = 1 for c in cases: self._dump_to_zk(c) cmt_txidset = self.storage.txidset.get()[COMMITTED] if max_txid == 1: self.assertEqual([], cmt_txidset) self.slave.apply() for k, v in c.items(): k_parts = k.split('/', 2) actual_val = self.storage.record.hget(k_parts[1], k_parts[2]) self.assertEqual(v, actual_val) ex = rangeset.RangeSet([[1, max_txid + 1]]) cmt_txidset = self.storage.txidset.get()[COMMITTED] self.assertEqual(ex, cmt_txidset) max_txid += 1 # no update self.slave.apply() ex = rangeset.RangeSet([[1, max_txid]]) cmt_txidset = self.storage.txidset.get()[COMMITTED] self.assertEqual(ex, cmt_txidset)
def purge(self, sets): topurge = rangeset.RangeSet() committed = sets[COMMITTED] l = committed.length() while l > self.max_journal_history: first = committed[0] # a range contains a single txid r = rangeset.RangeSet([[first[0], first[0] + 1]]) topurge.add(r[0]) committed = rangeset.substract(committed, r) l -= 1 for rng in topurge: for txid in range(rng[0], rng[1]): self.journal.safe_delete(txid) sets[PURGED] = rangeset.union(sets[PURGED], topurge) sets[COMMITTED] = rangeset.substract(sets[COMMITTED], topurge)
def test_init(self): a = rangeset.RangeSet() self.assertIsInstance(a, list) self.assertEqual(0, len(a)) a = rangeset.RangeSet([]) self.assertEqual(0, len(a))
def test_jour_not_found(self): c = { 'meta/server/k1': 1, 'meta/server/k2': 'server_v1', 'meta/server/k3': {}, 'meta/server/k4': { 'x': 'x' }, 'meta/server/k5': [], 'meta/server/k6': [1, 'x'], 'meta/region/k1': 1, 'meta/region/k2': 'region_v1', 'meta/region/k3': {}, 'meta/region/k4': { 'x': 'x' }, 'meta/region/k5': [], 'meta/region/k6': [1, 'x'], } self._dump_to_zk(c) self.zk.delete('tx/journal/0000000001') self.slave.apply() for k, v in c.items(): k_parts = k.split('/', 2) actual_val = self.storage.record.hget(k_parts[1], k_parts[2]) self.assertEqual(v, actual_val) ex = rangeset.RangeSet([[1, 2]]) cmt_txidset = self.storage.txidset.get()[COMMITTED] self.assertEqual(ex, cmt_txidset) # apply c = { 'meta/server/k1': 100, } self._dump_to_zk(c) self.slave.apply() ex = rangeset.RangeSet([[1, 3]]) cmt_txidset = self.storage.txidset.get()[COMMITTED] self.assertEqual(ex, cmt_txidset) actual_val = self.storage.record.hget('server', 'k1') self.assertEqual(100, actual_val)
def test_invalid_element_type(self): cases = ( int, long, float, str, tuple, list ) dd() for typ in cases: dd('test valid type: ', typ) rangeset.Range(typ(), typ()) rangeset.RangeSet([[typ(), typ()]]) cases = ( lambda x: 1, True, ) dd() for val in cases: dd('test invalid type: ', typ) self.assertRaises(TypeError, rangeset.Range, [val, val]) self.assertRaises(TypeError, rangeset.RangeSet, [[val, val]]) # incompatible type self.assertRaises(TypeError, rangeset.Range, 1, 'a')
def test_rangeset_has(self): rs = rangeset.RangeSet([[None, 1], [10, 20], [30, 40], [50, None]]) cases = ( (-1, True), (0, True), (1, False), (5, False), (9, False), (10, True), (15, True), (20, False), (21, False), (29, False), (30, True), (31, True), (40, False), (49, False), (50, True), (51, True), ) for val, expected in cases: dd('case:', val, expected) rst = rs.has(val) dd('rst:', rst) self.assertEqual(expected, rst)
def txidset_load(value): rst = {} for k in STATUS: if k not in value: value[k] = [] rst[k] = rangeset.RangeSet(value[k]) return rst
def test_int_inc_union(self): cases = ( ([[None, 10], [20, 30], [40, None]], [[None, None]], [[None, None]]), ([[None, 10], [20, 30], [40, None]], [[None, 1]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[None, 10]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[None, 11]], [[None, 11], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[9, 11]], [[None, 11], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[10, 11]], [[None, 11], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[11, 12]], [[None, 12], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[12, 13]], [[None, 10], [12, 13], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[18, 19]], [[None, 10], [18, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[19, 21]], [[None, 10], [19, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[20, 21]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[24, 25]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[29, 30]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[29, 31]], [[None, 10], [20, 31], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[30, 31]], [[None, 10], [20, 31], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[31, 32]], [[None, 10], [20, 32], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[32, 33]], [[None, 10], [20, 30], [32, 33], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[39, 39]], [[None, 10], [20, 30], [39, None]]), ([[None, 10], [20, 30], [40, None]], [[39, 40]], [[None, 10], [20, 30], [39, None]]), ([[None, 10], [20, 30], [40, None]], [[39, 41]], [[None, 10], [20, 30], [39, None]]), ([[None, 10], [20, 30], [40, None]], [[40, 41]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[41, 42]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[41, None]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[8, 25], [35, 40]], [[None, 30], [35, None]]), ) dd() for a, b, expected in cases: dd('case:', a, b, expected) a = rangeset.RangeSet(a, range_clz=rangeset.IntIncRange) b = rangeset.RangeSet(b, range_clz=rangeset.IntIncRange) rst = rangeset.union(a, b) dd('rst:', rst) self.assertEqual(expected, rst)
def test_int_inc_substract(self): cases = ( ([[None, 10], [20, 30], [40, None]], [[None, None]], []), ([[None, 10], [20, 30], [40, None]], [[None, 1]], [[2, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[None, 10]], [[20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[None, 11]], [[20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[9, 11]], [[None, 8], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[10, 11]], [[None, 9], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[11, 12]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[19, 20]], [[None, 10], [21, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[19, 21]], [[None, 10], [22, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[20, 21]], [[None, 10], [22, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[24, 25]], [[None, 10], [20, 23], [26, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[29, 30]], [[None, 10], [20, 28], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[29, 31]], [[None, 10], [20, 28], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[30, 31]], [[None, 10], [20, 29], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[31, 32]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[39, 40]], [[None, 10], [20, 30], [41, None]]), ([[None, 10], [20, 30], [40, None]], [[39, 41]], [[None, 10], [20, 30], [42, None]]), ([[None, 10], [20, 30], [40, None]], [[40, 41]], [[None, 10], [20, 30], [42, None]]), ([[None, 10], [20, 30], [40, None]], [[41, 42]], [[None, 10], [20, 30], [40, 40], [43, None]]), ([[None, 10], [20, 30], [40, None]], [[41, None]], [[None, 10], [20, 30], [40, 40], ]), ([[20, 30]], [[20, 23], [25, 30]], [[24, 24]]), ([[20, 30]], [[20, 22], [27, 30]], [[23, 26]]), ([[None, 10], [20, 30], [40, None]], [[20, 23], [26, 30]], [[None, 10], [24, 25], [40, None], ]), ([[None, 10], [20, 30], [40, None]], [[1, 2], [8, 25], [35, 45]], [[None, 0], [3, 7], [26, 30], [46, None]]), ) dd() for a, b, expected in cases: dd('case:', a, b, expected) a = rangeset.RangeSet(a, range_clz=rangeset.IntIncRange) b = rangeset.RangeSet(b, range_clz=rangeset.IntIncRange) rst = rangeset.substract(a, b) dd('rst:', rst) self.assertEqual(expected, rst)
def test_substract(self): cases = ( ([[None, 10], [20, 30], [40, None]], [[None, None]], []), ([[None, 10], [20, 30], [40, None]], [[None, 1]], [[1, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[None, 10]], [[20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[None, 11]], [[20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[9, 11]], [[None, 9], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[10, 11]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[11, 12]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[19, 20]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[19, 21]], [[None, 10], [21, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[20, 21]], [[None, 10], [21, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[24, 25]], [[None, 10], [20, 24], [25, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[29, 30]], [[None, 10], [20, 29], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[29, 31]], [[None, 10], [20, 29], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[30, 31]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[31, 32]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[39, 40]], [[None, 10], [20, 30], [40, None]]), ([[None, 10], [20, 30], [40, None]], [[39, 41]], [[None, 10], [20, 30], [41, None]]), ([[None, 10], [20, 30], [40, None]], [[40, 41]], [[None, 10], [20, 30], [41, None]]), ([[None, 10], [20, 30], [40, None]], [[41, 42]], [[None, 10], [20, 30], [40, 41], [42, None]]), ([[None, 10], [20, 30], [40, None]], [[41, None]], [[None, 10], [20, 30], [40, 41], ]), ([[20, 30]], [[20, 24], [25, 30]], [[24, 25]]), ([[None, 10], [20, 30], [40, None]], [[20, 24], [25, 30]], [[None, 10], [24, 25], [40, None], ]), ([[None, 10], [20, 30], [40, None]], [[1, 2], [8, 25], [35, 45]], [[None, 1], [2, 8], [25, 30], [45, None]]), ) dd() for a, b, expected in cases: dd('case:', a, b, expected) a = rangeset.RangeSet(a) b = rangeset.RangeSet(b) rst = rangeset.substract(a, b) dd('rst:', rst) self.assertEqual(expected, rst)
def txidset_load(value): val = utfjson.load(value) if val is None: val = {} committed = val.get(COMMITTED) or [] rst = {} rst[COMMITTED] = rangeset.RangeSet(committed) return rst
def test_int_add(self): cases = ( # add into empty range set. ([], [1, 1], [[1, 1]]), ([], [1, 2], [[1, 2]]), ([], [1, 3], [[1, 3]]), # collapse two range if necesary. ([[10, 20], [30, 40]], [1, 2], [[1, 2], [10, 20], [30, 40]]), ([[10, 20], [30, 40]], [1, 10], [[1, 20], [30, 40]]), ([[10, 20], [30, 40]], [1, 12], [[1, 20], [30, 40]]), ([[10, 20], [30, 40]], [10, 15], [[10, 20], [30, 40]]), ([[10, 20], [30, 40]], [11, 15], [[10, 20], [30, 40]]), ([[10, 20], [30, 40]], [1, 22], [[1, 22], [30, 40]]), ([[10, 20], [30, 40]], [15, 25], [[10, 25], [30, 40]]), ([[10, 20], [30, 40]], [20, 25], [[10, 25], [30, 40]]), ([[10, 20], [30, 40]], [22, 25], [[10, 20], [22, 25], [30, 40]]), ([[10, 20], [30, 40]], [22, 30], [[10, 20], [22, 40]]), ([[10, 20], [30, 40]], [22, 32], [[10, 20], [22, 40]]), ([[10, 20], [30, 40]], [30, 32], [[10, 20], [30, 40]]), ([[10, 20], [30, 40]], [30, 42], [[10, 20], [30, 42]]), ([[10, 20], [30, 40]], [32, 42], [[10, 20], [30, 42]]), ([[10, 20], [30, 40]], [40, 50], [[10, 20], [30, 50]]), ([[10, 20], [30, 40]], [42, 50], [[10, 20], [30, 40], [42, 50]]), # overlapping with more than one range ([[10, 20], [30, 40]], [20, 30], [[10, 40]]), ([[10, 20], [30, 40]], [19, 30], [[10, 40]]), ([[10, 20], [30, 40]], [20, 31], [[10, 40]]), ([[10, 20], [30, 40]], [0, 35], [[0, 40]]), ([[10, 20], [30, 40]], [15, 50], [[10, 50]]), ([[10, 20], [30, 40]], [15, 50], [[10, 50]]), ([[10, 20], [30, 40]], [15, None], [[10, None]]), ([[10, 20], [30, 40]], [None, 15], [[None, 20], [30, 40]]), ([[10, 20], [30, 40]], [None, 35], [[None, 40]]), ([[10, 20], [30, 40]], [None, None], [[None, None]]), ) dd() for init, ins, expected in cases: dd('cases: ', init, ins, expected) a = rangeset.RangeSet(init) a.add(ins) self.assertEqual(expected, a)
def test_find_overlapped(self): cases = ( ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [None, None], [[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [None, 1], [[None, 10, 'a'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [None, 10], [[None, 10, 'a'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [None, 11], [[None, 10, 'a'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [9, 11], [[None, 10, 'a'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [9, 21], [[None, 10, 'a'], [20, 30, 'b'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [9, 40], [[None, 10, 'a'], [20, 30, 'b'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [9, 41], [[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [10, 11], [ ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [11, 12], [ ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [19, 20], [ ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [19, 21], [ [20, 30, 'b'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [20, 21], [ [20, 30, 'b'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [24, 25], [ [20, 30, 'b'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [29, 30], [ [20, 30, 'b'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [29, 31], [ [20, 30, 'b'] ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [29, 41], [ [20, 30, 'b'], [40, None, 'c']]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [30, 31], [ ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [31, 32], [ ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [39, 40], [ ]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [39, 41], [ [40, None, 'c']]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [40, 41], [ [40, None, 'c']]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [41, 42], [ [40, None, 'c']]), ([[None, 10, 'a'], [20, 30, 'b'], [40, None, 'c']], [41, None], [ [40, None, 'c']]), ) dd() for a, b, expected in cases: dd('case:', a, b, expected) a = rangeset.RangeDict(a) self.assertEqual(expected, a.find_overlapped(b)) self.assertEqual(expected, a.find_overlapped(rangeset.Range(*b))) self.assertEqual(expected, a.find_overlapped(rangeset.ValueRange(*(b+['bar'])))) a = rangeset.RangeSet([x[:2] for x in a]) expected = [x[:2] for x in expected] self.assertEqual(expected, a.find_overlapped(b)) self.assertEqual(expected, a.find_overlapped(rangeset.Range(*b))) self.assertEqual(expected, a.find_overlapped(rangeset.ValueRange(*(b+['bar']))))
def test_int_add_error(self): cases = ( ([], None, TypeError), ([], True, TypeError), ([], {}, ValueError), ([], 1, TypeError), ([], 1.1, TypeError), ([], [1, 2, 3], TypeError), ([], lambda x: True, TypeError), ) dd() for init, ins, err in cases: dd('case: ', init, ins, err) a = rangeset.RangeSet(init) self.assertRaises(err, a.add, ins)
def test_range_incorrect_order(self): cases = ( [[None, None], [1, 2]], [[0, None], [1, 2]], [[1, 2], [None, 5]], [[1, 2], [2, 3]], [[1, 4], [3, 5]], [[3, 4], [1, 2]], ) for rs in cases: dd('case:', rs) try: rangeset.RangeSet(rs) except Exception as e: dd(repr(e)) self.assertRaises(ValueError, rangeset.RangeSet, rs)
def add_to_txidset(self, status, txid): if status not in STATUS: raise KeyError('invalid status: ' + repr(status)) logger.info('add {status}:{txid}' ' to txidset'.format(status=status, txid=txid)) for curr in txutil.cas_loop(self.txidset.get, self.txidset.set, conflicterror=self.conflicterror): for st in STATUS: if st not in curr.v: curr.v[st] = rangeset.RangeSet([]) curr.v[status].add([txid, txid + 1]) self.purge(curr.v)
def add_to_journal_id_set(self, status, journal_id): if status not in STATUS: raise KeyError('invalid status: ' + repr(status)) logger.info('add {st} journal id: {jid}'.format(st=status, jid=journal_id)) for curr in txutil.cas_loop(self.journal_id_set.get, self.journal_id_set.set, conflicterror=self.conflicterror): for st in STATUS: if st not in curr.v: curr.v[st] = rangeset.RangeSet([]) if status == COMMITTED: curr.v[status].add([0, int(journal_id) + 1]) else: curr.v[status].add([journal_id, int(journal_id) + 1]) self.purge(curr.v)
def purge(self, sets): committed_set = sets[COMMITTED] length = committed_set.length() if length <= self.max_journal_history: return purged_set = sets[PURGED] purged_end_jid = length - self.max_journal_history topurge = rangeset.RangeSet() for jid in range(purged_end_jid): if purged_set.has(jid): continue topurge.add([jid, jid + 1]) for rng in topurge: for jid in range(rng[0], rng[1]): self.journal.safe_delete('{jid:0>10}'.format(jid=jid)) sets[PURGED] = rangeset.union(sets[PURGED], topurge)
def test_purge(self): self.sto.max_journal_history = 5 cases = ( ( {PURGED: [], COMMITTED: [[0, 100]]}, {PURGED: [[0, 95]], COMMITTED: [[0, 100]]}, ), ( {PURGED: [[1, 10]], COMMITTED: [[0, 200]]}, {PURGED: [[0, 195]], COMMITTED: [[0, 200]]}, ), # no need to purge ( {PURGED: [[0, 95]], COMMITTED: [[0, 100]]}, {PURGED: [[0, 95]], COMMITTED: [[0, 100]]}, ), ) for inp, expected in cases: inp = {k: rangeset.RangeSet(v) for k, v in inp.items()} self.sto.purge(inp) self.assertEqual(expected, inp)
def test_length(self): rst = rangeset.RangeSet([[1, 2], [5, 8]]).length() self.assertEqual(4, rst) rst = rangeset.RangeSet([['a', 'b'], ['d', 'd\0']]).length() self.assertEqual(1.0/257.0 + 1.0/257.0/257.0, rst)
def get(self): with self.lock: for k in self.d: self.d[k] = rangeset.RangeSet(self.d[k]) return self.d, self.ver
def test_int_compare(self): a = rangeset.RangeSet([]) self.assertEqual([], a)
print r1.has('wo') print r1.has([1, 2, 4, 5]) print print r1.is_adjacent(r2) print r2.is_adjacent(r1) print r1.is_adjacent(r3) print r1.is_adjacent(r4) print print r1.length() print r2.length() print # r1 = rangeset.RangeSet([[1, 5], [4, 6], [10, 20]]) r1 = rangeset.RangeSet([[1, 5], [6, 10], [15, 20]]) print r1 print print r1.has(10) print r1.add([5, 6]) print r1 print print r1.has(10) print r1 = rangeset.IntIncRange(1, 5) r2 = rangeset.IntIncRange(7, 10)