def getObject(self, oid, tid=None, before_tid=None): """ oid (packed) Identifier of object to retrieve. tid (packed, None) Exact serial to retrieve. before_tid (packed, None) Serial to retrieve is the highest existing one strictly below this value. Return value: None: Given oid doesn't exist in database. False: No record found, but another one exists for given oid. 6-tuple: Record content. - record serial (packed) - serial or next record modifying object (packed, None) - compression (boolean-ish, None) - checksum (integer, None) - data (binary string, None) - data_serial (packed, None) """ u64 = util.u64 r = self._getObject(u64(oid), tid and u64(tid), before_tid and u64(before_tid)) try: serial, next_serial, compression, checksum, data, data_serial = r except TypeError: # See if object exists at all return (tid or before_tid) and self.getLastObjectTID(oid) and False return (util.p64(serial), None if next_serial is None else util.p64(next_serial), compression, checksum, data, None if data_serial is None else util.p64(data_serial))
def test_max_allowed_packet(self): EXTRA = 2 # Check EXTRA x = "SELECT '%s'" % ('x' * (self.db._max_allowed_packet - 11)) assert len(x) + EXTRA == self.db._max_allowed_packet self.assertRaises(DatabaseFailure, self.db.query, x + ' ') self.db.query(x) # Check MySQLDatabaseManager._max_allowed_packet query_list = [] query = self.db.query self.db.query = lambda query: query_list.append(EXTRA + len(query)) self.assertEqual(2, max(len(self.db.escape(chr(x))) for x in xrange(256))) self.assertEqual(2, len(self.db.escape('\0'))) self.db.storeData('\0' * 20, '\0' * (2**24 - 1), 0) size, = query_list max_allowed = self.db.__class__._max_allowed_packet self.assertTrue(max_allowed - 1024 < size <= max_allowed, size) # Check storeTransaction for count, max_allowed_packet in (7, 64), (6, 65), (1, 215): self.db._max_allowed_packet = max_allowed_packet del query_list[:] self.db.storeTransaction(p64(0), ((p64(1 << i), 0, None) for i in xrange(10)), None) self.assertEqual(max(query_list), max_allowed_packet) self.assertEqual(len(query_list), count)
def corrupt(offset): s0, s1, s2 = (storage_dict[cell.getUUID()] for cell in cluster.master.pt.getCellList(offset, True)) logging.info('corrupt partition %u of %s', offset, uuid_str(s1.uuid)) s1.dm.deleteObject(p64(np+offset), p64(corrupt_tid)) return s0.uuid
def test_checkRange(self): def check(trans, obj, *args): self.assertEqual(trans, self.db.checkTIDRange(*args)) self.assertEqual(obj, self.db.checkSerialRange(*(args + (ZERO_OID, )))) self.setNumPartitions(2, True) tid1, tid2, tid3, tid4 = self._storeTransactions(4) z = 0, ZERO_HASH, ZERO_TID, ZERO_HASH, ZERO_OID # - one partition check((2, a2b_hex('84320eb8dbbe583f67055c15155ab6794f11654d'), tid3), z, 0, 10, ZERO_TID, MAX_TID) # - another partition check((2, a2b_hex('1f02f98cf775a9e0ce9252ff5972dce728c4ddb0'), tid4), (4, a2b_hex('e5b47bddeae2096220298df686737d939a27d736'), tid4, a2b_hex('1e9093698424b5370e19acd2d5fc20dcd56a32cd'), p64(1)), 1, 10, ZERO_TID, MAX_TID) self.assertEqual( (3, a2b_hex('b85e2d4914e22b5ad3b82b312b3dc405dc17dcb8'), tid4, a2b_hex('1b6d73ecdc064595fe915a5c26da06b195caccaa'), p64(1)), self.db.checkSerialRange(1, 10, ZERO_TID, MAX_TID, p64(2))) # - min_tid is inclusive check((1, a2b_hex('da4b9237bacccdf19c0760cab7aec4a8359010b0'), tid3), z, 0, 10, tid3, MAX_TID) # - max tid is inclusive x = 1, a2b_hex('b6589fc6ab0dc82cf12099d1c2d40ab994e8410c'), tid1 check(x, z, 0, 10, ZERO_TID, tid2) # - limit y = 1, a2b_hex('356a192b7913b04c54574d18c28d46e6395428ab'), tid2 check(y, x + y[1:], 1, 1, ZERO_TID, MAX_TID)
def test_max_allowed_packet(self): EXTRA = 2 # Check EXTRA x = "SELECT '%s'" % ('x' * (self.db._max_allowed_packet - 11)) assert len(x) + EXTRA == self.db._max_allowed_packet self.assertRaises(DatabaseFailure, self.db.query, x + ' ') self.db.query(x) # Reconnection cleared the cache of the config table, # so fill it again with required values before we patch query(). self.db._getPartition # Check MySQLDatabaseManager._max_allowed_packet query_list = [] self.db.query = lambda query: query_list.append(EXTRA + len(query)) self.assertEqual(2, max(len(self.db.escape(chr(x))) for x in xrange(256))) self.assertEqual(2, len(self.db.escape('\0'))) self.db.storeData('\0' * 20, ZERO_OID, '\0' * (2**24 - 1), 0) size, = query_list max_allowed = self.db.__class__._max_allowed_packet self.assertTrue(max_allowed - 1024 < size <= max_allowed, size) # Check storeTransaction for count, max_allowed_packet in (7, 64), (6, 65), (1, 215): self.db._max_allowed_packet = max_allowed_packet del query_list[:] self.db.storeTransaction(p64(0), ((p64(1 << i), 1234, None) for i in xrange(10)), None) self.assertEqual(max(query_list), max_allowed_packet) self.assertEqual(len(query_list), count)
def test_checkRange(self): def check(trans, obj, *args): self.assertEqual(trans, self.db.checkTIDRange(*args)) self.assertEqual(obj, self.db.checkSerialRange(*(args+(ZERO_OID,)))) self.setNumPartitions(2, True) tid1, tid2, tid3, tid4 = self._storeTransactions(4) z = 0, ZERO_HASH, ZERO_TID, ZERO_HASH, ZERO_OID # - one partition check((2, a2b_hex('84320eb8dbbe583f67055c15155ab6794f11654d'), tid3), z, 0, 10, ZERO_TID, MAX_TID) # - another partition check((2, a2b_hex('1f02f98cf775a9e0ce9252ff5972dce728c4ddb0'), tid4), (4, a2b_hex('e5b47bddeae2096220298df686737d939a27d736'), tid4, a2b_hex('1e9093698424b5370e19acd2d5fc20dcd56a32cd'), p64(1)), 1, 10, ZERO_TID, MAX_TID) self.assertEqual( (3, a2b_hex('b85e2d4914e22b5ad3b82b312b3dc405dc17dcb8'), tid4, a2b_hex('1b6d73ecdc064595fe915a5c26da06b195caccaa'), p64(1)), self.db.checkSerialRange(1, 10, ZERO_TID, MAX_TID, p64(2))) # - min_tid is inclusive check((1, a2b_hex('da4b9237bacccdf19c0760cab7aec4a8359010b0'), tid3), z, 0, 10, tid3, MAX_TID) # - max tid is inclusive x = 1, a2b_hex('b6589fc6ab0dc82cf12099d1c2d40ab994e8410c'), tid1 check(x, z, 0, 10, ZERO_TID, tid2) # - limit y = 1, a2b_hex('356a192b7913b04c54574d18c28d46e6395428ab'), tid2 check(y, x + y[1:], 1, 1, ZERO_TID, MAX_TID)
def test_max_allowed_packet(self): EXTRA = 2 # Check EXTRA x = "SELECT '%s'" % ('x' * (self.db._max_allowed_packet - 11)) assert len(x) + EXTRA == self.db._max_allowed_packet self.assertRaises(DatabaseFailure, self.db.query, x + ' ') self.db.query(x) # Check MySQLDatabaseManager._max_allowed_packet query_list = [] query = self.db.query self.db.query = lambda query: query_list.append(EXTRA + len(query)) self.assertEqual(2, max(len(self.db.escape(chr(x))) for x in xrange(256))) self.assertEqual(2, len(self.db.escape('\0'))) self.db.storeData('\0' * 20, '\0' * (2**24-1), 0) size, = query_list max_allowed = self.db.__class__._max_allowed_packet self.assertTrue(max_allowed - 1024 < size <= max_allowed, size) # Check storeTransaction for count, max_allowed_packet in (7, 64), (6, 65), (1, 215): self.db._max_allowed_packet = max_allowed_packet del query_list[:] self.db.storeTransaction(p64(0), ((p64(1<<i),0,None) for i in xrange(10)), None) self.assertEqual(max(query_list), max_allowed_packet) self.assertEqual(len(query_list), count)
def corrupt(offset): s0, s1, s2 = ( storage_dict[cell.getUUID()] for cell in cluster.master.pt.getCellList(offset, True)) logging.info('corrupt partition %u of %s', offset, uuid_str(s1.uuid)) s1.dm.deleteObject(p64(np + offset), p64(corrupt_tid)) return s0.uuid
def testCheckReplicas(self): from neo.storage import checker def corrupt(offset): s0, s1, s2 = ( storage_dict[cell.getUUID()] for cell in cluster.master.pt.getCellList(offset, True)) logging.info('corrupt partition %u of %s', offset, uuid_str(s1.uuid)) s1.dm.deleteObject(p64(np + offset), p64(corrupt_tid)) return s0.uuid def check(expected_state, expected_count): self.assertEqual( expected_count, len([ None for row in cluster.neoctl.getPartitionRowList()[1] for cell in row[1] if cell[1] == CellStates.CORRUPTED ])) self.assertEqual(expected_state, cluster.neoctl.getClusterState()) np = 5 tid_count = np * 3 corrupt_tid = tid_count // 2 check_dict = dict.fromkeys(xrange(np)) cluster = NEOCluster(partitions=np, replicas=2, storage_count=3) try: checker.CHECK_COUNT = 2 cluster.start() cluster.populate([range(np * 2)] * tid_count) storage_dict = {x.uuid: x for x in cluster.storage_list} cluster.neoctl.checkReplicas(check_dict, ZERO_TID, None) self.tic() check(ClusterStates.RUNNING, 0) source = corrupt(0) cluster.neoctl.checkReplicas(check_dict, p64(corrupt_tid + 1), None) self.tic() check(ClusterStates.RUNNING, 0) cluster.neoctl.checkReplicas({0: source}, ZERO_TID, None) self.tic() check(ClusterStates.RUNNING, 1) corrupt(1) cluster.neoctl.checkReplicas(check_dict, p64(corrupt_tid + 1), None) self.tic() check(ClusterStates.RUNNING, 1) cluster.neoctl.checkReplicas(check_dict, ZERO_TID, None) self.tic() check(ClusterStates.RECOVERING, 4) finally: checker.CHECK_COUNT = CHECK_COUNT cluster.stop()
def populate(self, transaction_list, tid=lambda i: p64(i + 1), oid=lambda i: p64(i + 1)): storage = self.getZODBStorage() tid_dict = {} for i, oid_list in enumerate(transaction_list): txn = transaction.Transaction() storage.tpc_begin(txn, tid(i)) for o in oid_list: storage.store(oid(o), tid_dict.get(o), repr((i, o)), "", txn) storage.tpc_vote(txn) i = storage.tpc_finish(txn) for o in oid_list: tid_dict[o] = i
def populate(self, transaction_list, tid=lambda i: p64(i+1), oid=lambda i: p64(i+1)): storage = self.getZODBStorage() tid_dict = {} for i, oid_list in enumerate(transaction_list): txn = transaction.Transaction() storage.tpc_begin(txn, tid(i)) for o in oid_list: storage.store(oid(o), tid_dict.get(o), repr((i, o)), '', txn) storage.tpc_vote(txn) i = storage.tpc_finish(txn) for o in oid_list: tid_dict[o] = i
def fetchObject(self, oid, tid): """ Specialized version of getObject, for replication: - the oid can only be at an exact serial (parameter 'tid') - next_serial is not part of the result - if there's no result for the requested serial, no need check if oid exists at other serial """ u64 = util.u64 r = self._fetchObject(u64(oid), u64(tid)) if r: serial, compression, checksum, data, data_serial = r return (util.p64(serial), compression, checksum, data, None if data_serial is None else util.p64(data_serial))
def _nextTID(orig, tm, ttid=None, divisor=None): n = self._next_tid self._next_tid = n + 1 n = str(n).rjust(3, '-') if ttid: t = u64('T%s%s-' % (n, ttid[1:4])) m = (u64(ttid) - t) % divisor assert m < 211, (p64(t), divisor) t = p64(t + m) else: t = 'T%s----' % n assert tm._last_tid < t, (tm._last_tid, t) tm._last_tid = t return t
def getLastObjectTID(self, oid): oid = util.u64(oid) r = self.query("SELECT tid FROM obj" " WHERE partition=? AND oid=?" " ORDER BY tid DESC LIMIT 1", (self._getPartition(oid), oid)).fetchone() return r and util.p64(r[0])
def getLastObjectTID(self, oid): oid = util.u64(oid) r = self.query("SELECT tid FROM obj" " WHERE partition=? AND oid=?" " ORDER BY tid DESC LIMIT 1", (self._getReadablePartition(oid), oid)).fetchone() return r and util.p64(r[0])
def moduloTID(self, partition): """Force generation of TIDs that will be stored in given partition""" partition = p64(partition) master = self.primary_master return Patch( master.tm, _nextTID=lambda orig, *args: orig(*args) if args else orig(partition, master.pt.getPartitions()) )
def getLastObjectTID(self, oid): oid = util.u64(oid) r = self.query("SELECT tid FROM obj FORCE INDEX(PRIMARY)" " WHERE `partition`=%d AND oid=%d" " ORDER BY tid DESC LIMIT 1" % (self._getReadablePartition(oid), oid)) return util.p64(r[0][0]) if r else None
def testClientReadingDuringTweak(self): # XXX: Currently, the test passes because data of dropped cells are not # deleted while the cluster is operational: this is only done # during the RECOVERING phase. But we'll want to be able to free # disk space without service interruption, and for this the client # may have to retry reading data from the new cells. If s0 deleted # all data for partition 1, the test would fail with a POSKeyError. cluster = NEOCluster(partitions=2, storage_count=2) s0, s1 = cluster.storage_list try: cluster.start([s0]) storage = cluster.getZODBStorage() oid = p64(1) txn = transaction.Transaction() storage.tpc_begin(txn) storage.store(oid, None, 'foo', '', txn) storage.tpc_finish(txn) storage._cache.clear() s1.start() self.tic() cluster.neoctl.enableStorageList([s1.uuid]) cluster.neoctl.tweakPartitionTable() with cluster.master.filterConnection(cluster.client) as m2c: m2c.add(lambda conn, packet: isinstance(packet, Packets.NotifyPartitionChanges)) self.tic() self.assertEqual('foo', storage.load(oid)[0]) finally: cluster.stop()
def moduloTID(self, partition): """Force generation of TIDs that will be stored in given partition""" partition = p64(partition) master = self.primary_master return Patch(master.tm, _nextTID=lambda orig, *args: orig(*args) if args else orig(partition, master.pt.getPartitions()))
def getLastIDs(self): """Return max(tid) & max(oid) for readable data It is important to ignore unassigned partitions because there may remain data from cells that have been discarded, either due to --disable-drop-partitions option, or in the future when dropping partitions is done in background (as it is an expensive operation). """ x = self._readable_set if x: tid, oid = zip(*map(self._getLastIDs, x)) tid = max(self.getLastTID(), max(tid)) oid = max(oid) return (None if tid is None else util.p64(tid), None if oid is None else util.p64(oid)) return None, None
def getLastObjectTID(self, oid): oid = util.u64(oid) r = self.query("SELECT tid FROM obj" " WHERE `partition`=%d AND oid=%d" " ORDER BY tid DESC LIMIT 1" % (self._getPartition(oid), oid)) return util.p64(r[0][0]) if r else None
def testClientReadingDuringTweak(self, cluster): def sync(orig): m2c.remove(delay) orig() s0, s1 = cluster.storage_list if 1: cluster.start([s0]) storage = cluster.getZODBStorage() oid = p64(1) txn = transaction.Transaction() storage.tpc_begin(txn) storage.store(oid, None, 'foo', '', txn) storage.tpc_finish(txn) storage._cache.clear() s1.start() self.tic() cluster.neoctl.enableStorageList([s1.uuid]) cluster.neoctl.tweakPartitionTable() with cluster.master.filterConnection(cluster.client) as m2c: delay = m2c.delayNotifyPartitionChanges() self.tic() with Patch(cluster.client, sync=sync): self.assertEqual('foo', storage.load(oid)[0]) self.assertNotIn(delay, m2c)
def _storeTransactions(self, count): # use OID generator to know result of tid % N tid_list = self.getOIDs(count) oid = p64(1) for tid in tid_list: txn, objs = self.getTransaction([oid]) self.db.storeTransaction(tid, objs, txn, False) return tid_list
def test_getTransaction(self): oid1, oid2 = self.getOIDs(2) tid1, tid2 = self.getTIDs(2) txn1, objs1 = self.getTransaction([oid1]) txn2, objs2 = self.getTransaction([oid2]) # get from temporary table or not with self.commitTransaction(tid1, objs1, txn1), \ self.commitTransaction(tid2, objs2, txn2, None): pass result = self.db.getTransaction(tid1, True) self.assertEqual(result, ([oid1], 'user', 'desc', 'ext', False, p64(1))) result = self.db.getTransaction(tid2, True) self.assertEqual(result, ([oid2], 'user', 'desc', 'ext', False, p64(2))) # get from non-temporary only result = self.db.getTransaction(tid1, False) self.assertEqual(result, ([oid1], 'user', 'desc', 'ext', False, p64(1))) self.assertEqual(self.db.getTransaction(tid2, False), None)
def getFinalTID(self, ttid): ttid = util.u64(ttid) # As of SQLite 3.8.7.1, 'tid>=ttid' would ignore the index on tid, # even though ttid is a constant. for tid, in self.query("SELECT tid FROM trans" " WHERE partition=? AND tid>=? AND ttid=? LIMIT 1", (self._getPartition(ttid), ttid, ttid)): return util.p64(tid)
def getFinalTID(self, ttid): ttid = util.u64(ttid) # MariaDB is smart enough to realize that 'ttid' is constant. r = self.query("SELECT tid FROM trans" " WHERE `partition`=%s AND tid>=ttid AND ttid=%s LIMIT 1" % (self._getPartition(ttid), ttid)) if r: return util.p64(r[0][0])
def getFinalTID(self, ttid): ttid = util.u64(ttid) # As of SQLite 3.8.7.1, 'tid>=ttid' would ignore the index on tid, # even though ttid is a constant. for tid, in self.query("SELECT tid FROM trans" " WHERE partition=? AND tid>=? AND ttid=? LIMIT 1", (self._getReadablePartition(ttid), ttid, ttid)): return util.p64(tid)
def testCheckReplicas(self): from neo.storage import checker def corrupt(offset): s0, s1, s2 = (storage_dict[cell.getUUID()] for cell in cluster.master.pt.getCellList(offset, True)) logging.info('corrupt partition %u of %s', offset, uuid_str(s1.uuid)) s1.dm.deleteObject(p64(np+offset), p64(corrupt_tid)) return s0.uuid def check(expected_state, expected_count): self.assertEqual(expected_count, len([None for row in cluster.neoctl.getPartitionRowList()[1] for cell in row[1] if cell[1] == CellStates.CORRUPTED])) self.assertEqual(expected_state, cluster.neoctl.getClusterState()) np = 5 tid_count = np * 3 corrupt_tid = tid_count // 2 check_dict = dict.fromkeys(xrange(np)) cluster = NEOCluster(partitions=np, replicas=2, storage_count=3) try: checker.CHECK_COUNT = 2 cluster.start() cluster.populate([range(np*2)] * tid_count) storage_dict = {x.uuid: x for x in cluster.storage_list} cluster.neoctl.checkReplicas(check_dict, ZERO_TID, None) self.tic() check(ClusterStates.RUNNING, 0) source = corrupt(0) cluster.neoctl.checkReplicas(check_dict, p64(corrupt_tid+1), None) self.tic() check(ClusterStates.RUNNING, 0) cluster.neoctl.checkReplicas({0: source}, ZERO_TID, None) self.tic() check(ClusterStates.RUNNING, 1) corrupt(1) cluster.neoctl.checkReplicas(check_dict, p64(corrupt_tid+1), None) self.tic() check(ClusterStates.RUNNING, 1) cluster.neoctl.checkReplicas(check_dict, ZERO_TID, None) self.tic() check(ClusterStates.RECOVERING, 4) finally: checker.CHECK_COUNT = CHECK_COUNT cluster.stop()
def getFinalTID(self, ttid): ttid = util.u64(ttid) # MariaDB is smart enough to realize that 'ttid' is constant. r = self.query( "SELECT tid FROM trans" " WHERE `partition`=%s AND tid>=ttid AND ttid=%s LIMIT 1" % (self._getReadablePartition(ttid), ttid)) if r: return util.p64(r[0][0])
def test_findUndoTID(self): self.setNumPartitions(4, True) db = self.db tid1 = self.getNextTID() tid2 = self.getNextTID() tid3 = self.getNextTID() tid4 = self.getNextTID() tid5 = self.getNextTID() oid1 = p64(1) foo = db.holdData("3" * 20, oid1, 'foo', 0) bar = db.holdData("4" * 20, oid1, 'bar', 0) db.releaseData((foo, bar)) db.storeTransaction(tid1, ((oid1, foo, None), ), None, temporary=False) # Undoing oid1 tid1, OK: tid1 is latest # Result: current tid is tid1, data_tid is None (undoing object # creation) self.assertEqual(db.findUndoTID(oid1, tid5, tid4, tid1, None), (tid1, None, True)) # Store a new transaction db.storeTransaction(tid2, ((oid1, bar, None), ), None, temporary=False) # Undoing oid1 tid2, OK: tid2 is latest # Result: current tid is tid2, data_tid is tid1 self.assertEqual(db.findUndoTID(oid1, tid5, tid4, tid2, None), (tid2, tid1, True)) # Undoing oid1 tid1, Error: tid2 is latest # Result: current tid is tid2, data_tid is -1 self.assertEqual(db.findUndoTID(oid1, tid5, tid4, tid1, None), (tid2, None, False)) # Undoing oid1 tid1 with tid2 being undone in same transaction, # OK: tid1 is latest # Result: current tid is tid1, data_tid is None (undoing object # creation) # Explanation of transaction_object: oid1, no data but a data serial # to tid1 self.assertEqual( db.findUndoTID(oid1, tid5, tid4, tid1, (u64(oid1), None, tid1)), (tid1, None, True)) # Store a new transaction db.storeTransaction(tid3, ((oid1, None, tid1), ), None, temporary=False) # Undoing oid1 tid1, OK: tid3 is latest with tid1 data # Result: current tid is tid2, data_tid is None (undoing object # creation) self.assertEqual(db.findUndoTID(oid1, tid5, tid4, tid1, None), (tid3, None, True))
def checkTIDRange(self, partition, length, min_tid, max_tid): # XXX: SQLite's GROUP_CONCAT is slow (looks like quadratic) count, tids, max_tid = self.query("""\ SELECT COUNT(*), GROUP_CONCAT(tid), MAX(tid) FROM (SELECT tid FROM trans WHERE partition=? AND ?<=tid AND tid<=? ORDER BY tid ASC LIMIT ?) AS t""", (partition, util.u64(min_tid), util.u64(max_tid), -1 if length is None else length)).fetchone() if count: return count, sha1(tids).digest(), util.p64(max_tid) return 0, ZERO_HASH, ZERO_TID
def test_commitTransaction(self): oid1, oid2 = self.getOIDs(2) tid1, tid2 = self.getTIDs(2) txn1, objs1 = self.getTransaction([oid1]) txn2, objs2 = self.getTransaction([oid2]) # nothing in database self.assertEqual(self.db.getLastIDs(), (None, None)) self.assertEqual(self.db.getUnfinishedTIDDict(), {}) self.assertEqual(self.db.getObject(oid1), None) self.assertEqual(self.db.getObject(oid2), None) self.assertEqual(self.db.getTransaction(tid1, True), None) self.assertEqual(self.db.getTransaction(tid2, True), None) self.assertEqual(self.db.getTransaction(tid1, False), None) self.assertEqual(self.db.getTransaction(tid2, False), None) with self.commitTransaction(tid1, objs1, txn1), \ self.commitTransaction(tid2, objs2, txn2): self.assertEqual(self.db.getTransaction(tid1, True), ([oid1], 'user', 'desc', 'ext', False, p64(1))) self.assertEqual(self.db.getTransaction(tid2, True), ([oid2], 'user', 'desc', 'ext', False, p64(2))) self.assertEqual(self.db.getTransaction(tid1, False), None) self.assertEqual(self.db.getTransaction(tid2, False), None) result = self.db.getTransaction(tid1, True) self.assertEqual(result, ([oid1], 'user', 'desc', 'ext', False, p64(1))) result = self.db.getTransaction(tid2, True) self.assertEqual(result, ([oid2], 'user', 'desc', 'ext', False, p64(2))) result = self.db.getTransaction(tid1, False) self.assertEqual(result, ([oid1], 'user', 'desc', 'ext', False, p64(1))) result = self.db.getTransaction(tid2, False) self.assertEqual(result, ([oid2], 'user', 'desc', 'ext', False, p64(2)))
def test_commitTransaction(self): oid1, oid2 = self.getOIDs(2) tid1, tid2 = self.getTIDs(2) txn1, objs1 = self.getTransaction([oid1]) txn2, objs2 = self.getTransaction([oid2]) # nothing in database self.assertEqual(self.db.getLastIDs(), (None, {}, {}, None)) self.assertEqual(self.db.getUnfinishedTIDDict(), {}) self.assertEqual(self.db.getObject(oid1), None) self.assertEqual(self.db.getObject(oid2), None) self.assertEqual(self.db.getTransaction(tid1, True), None) self.assertEqual(self.db.getTransaction(tid2, True), None) self.assertEqual(self.db.getTransaction(tid1, False), None) self.assertEqual(self.db.getTransaction(tid2, False), None) with self.commitTransaction(tid1, objs1, txn1), \ self.commitTransaction(tid2, objs2, txn2): self.assertEqual(self.db.getTransaction(tid1, True), ([oid1], 'user', 'desc', 'ext', False, p64(1))) self.assertEqual(self.db.getTransaction(tid2, True), ([oid2], 'user', 'desc', 'ext', False, p64(2))) self.assertEqual(self.db.getTransaction(tid1, False), None) self.assertEqual(self.db.getTransaction(tid2, False), None) result = self.db.getTransaction(tid1, True) self.assertEqual(result, ([oid1], 'user', 'desc', 'ext', False, p64(1))) result = self.db.getTransaction(tid2, True) self.assertEqual(result, ([oid2], 'user', 'desc', 'ext', False, p64(2))) result = self.db.getTransaction(tid1, False) self.assertEqual(result, ([oid1], 'user', 'desc', 'ext', False, p64(1))) result = self.db.getTransaction(tid2, False) self.assertEqual(result, ([oid2], 'user', 'desc', 'ext', False, p64(2)))
def getTransaction(self, tid, all=False): tid = util.u64(tid) q = self.query r = q("SELECT oids, user, description, ext, packed, ttid" " FROM trans WHERE partition=? AND tid=?", (self._getReadablePartition(tid), tid)).fetchone() if not r and all: r = q("SELECT oids, user, description, ext, packed, ttid" " FROM ttrans WHERE tid=?", (tid,)).fetchone() if r: oids, user, description, ext, packed, ttid = r return splitOIDField(tid, oids), str(user), \ str(description), str(ext), packed, util.p64(ttid)
def getTransaction(self, tid, all = False): tid = util.u64(tid) q = self.query r = q("SELECT oids, user, description, ext, packed, ttid" " FROM trans WHERE `partition` = %d AND tid = %d" % (self._getPartition(tid), tid)) if not r and all: r = q("SELECT oids, user, description, ext, packed, ttid" " FROM ttrans WHERE tid = %d" % tid) if r: oids, user, desc, ext, packed, ttid = r[0] oid_list = splitOIDField(tid, oids) return oid_list, user, desc, ext, bool(packed), util.p64(ttid)
def getTransaction(self, tid, all=False): tid = util.u64(tid) q = self.query r = q("SELECT oids, user, description, ext, packed, ttid" " FROM trans WHERE partition=? AND tid=?", (self._getPartition(tid), tid)).fetchone() if not r and all: r = q("SELECT oids, user, description, ext, packed, ttid" " FROM ttrans WHERE tid=?", (tid,)).fetchone() if r: oids, user, description, ext, packed, ttid = r return splitOIDField(tid, oids), str(user), \ str(description), str(ext), packed, util.p64(ttid)
def getTransaction(self, tid, all=False): tid = util.u64(tid) q = self.query r = q("SELECT oids, user, description, ext, packed, ttid" " FROM trans WHERE `partition` = %d AND tid = %d" % (self._getReadablePartition(tid), tid)) if not r and all: r = q("SELECT oids, user, description, ext, packed, ttid" " FROM ttrans WHERE tid = %d" % tid) if r: oids, user, desc, ext, packed, ttid = r[0] oid_list = splitOIDField(tid, oids) return oid_list, user, desc, ext, bool(packed), util.p64(ttid)
def getDataTid(self, oid, tid): try: return self.data_tid[tid].get(oid) except KeyError: assert tid not in self.data_tid, (oid, tid) p_tid = util.p64(tid) txn = next(self.storage.iterator(p_tid)) if txn.tid != p_tid: raise u64 = util.u64 txn = self.data_tid[tid] = { u64(x.oid): x.data_txn for x in txn if x.data_txn} return txn.get(oid)
def getDataTid(self, oid, tid): try: return self.data_tid[tid].get(oid) except KeyError: assert tid not in self.data_tid, (oid, tid) p_tid = util.p64(tid) txn = next(self.storage.iterator(p_tid)) if txn.tid != p_tid: raise u64 = util.u64 txn = self.data_tid[tid] = { u64(x.oid): x.data_txn for x in txn if x.data_txn } return txn.get(oid)
def checkTIDRange(self, partition, length, min_tid, max_tid): count, tid_checksum, max_tid = self.query( """SELECT COUNT(*), SHA1(GROUP_CONCAT(tid SEPARATOR ",")), MAX(tid) FROM (SELECT tid FROM trans WHERE `partition` = %(partition)s AND tid >= %(min_tid)d AND tid <= %(max_tid)d ORDER BY tid ASC %(limit)s) AS t""" % { 'partition': partition, 'min_tid': util.u64(min_tid), 'max_tid': util.u64(max_tid), 'limit': '' if length is None else 'LIMIT %u' % length, })[0] if count: return count, a2b_hex(tid_checksum), util.p64(max_tid) return 0, ZERO_HASH, ZERO_TID
def getObject(self, oid, tid=None, before_tid=None): u64 = util.u64 u_oid = u64(oid) u_tid = tid and u64(tid) u_before_tid = before_tid and u64(before_tid) db = self.db if self.zodb_tid < (u_before_tid - 1 if before_tid else u_tid or 0) <= self.zodb_ltid: o = None else: o = db.getObject(oid, tid, before_tid) if o and self.zodb_ltid < u64(o[0]) or \ not self.inZodb(u_oid, u_tid, u_before_tid): return o p64 = util.p64 zodb, z_oid = self.zodbFromOid(u_oid) try: value, serial, next_serial = zodb.loadBefore( p64(z_oid), before_tid or (util.p64(u_tid + 1) if tid else MAX_TID)) except TypeError: # loadBefore returned None return False except POSKeyError: assert not o, o return o if serial != tid: if tid: return False u_tid = u64(serial) if u_tid <= self.zodb_tid and o: return o if value: value = zodb.repickle(value) checksum = util.makeChecksum(value) else: # CAVEAT: Although we think loadBefore should not return an empty # value for a deleted object (BBB: fixed in ZODB4), # there's no need to distinguish this case in the above # except clause because it would be crazy to import a # NEO DB using this backend. checksum = None if not next_serial: next_serial = db._getNextTID(db._getPartition(u_oid), u_oid, u_tid) if next_serial: next_serial = p64(next_serial) return (serial, next_serial, 0, checksum, value, zodb.getDataTid(z_oid, u_tid))
def getObject(self, oid, tid=None, before_tid=None): u64 = util.u64 u_oid = u64(oid) u_tid = tid and u64(tid) u_before_tid = before_tid and u64(before_tid) db = self.db if self.zodb_tid < (u_before_tid - 1 if before_tid else u_tid or 0) <= self.zodb_ltid: o = None else: o = db.getObject(oid, tid, before_tid) if o and self.zodb_ltid < u64(o[0]) or \ not self.inZodb(u_oid, u_tid, u_before_tid): return o p64 = util.p64 zodb, z_oid = self.zodbFromOid(u_oid) try: value, serial, next_serial = zodb.loadBefore(p64(z_oid), before_tid or (util.p64(u_tid + 1) if tid else MAX_TID)) except TypeError: # loadBefore returned None return False except POSKeyError: assert not o, o return o if serial != tid: if tid: return False u_tid = u64(serial) if u_tid <= self.zodb_tid and o: return o if value: value = zodb.repickle(value) checksum = util.makeChecksum(value) else: # CAVEAT: Although we think loadBefore should not return an empty # value for a deleted object (see comment in NEO Storage), # there's no need to distinguish this case in the above # except clause because it would be crazy to import a # NEO DB using this backend. checksum = None if not next_serial: next_serial = db._getNextTID(db._getPartition(u_oid), u_oid, u_tid) if next_serial: next_serial = p64(next_serial) return (serial, next_serial, 0, checksum, value, zodb.getDataTid(z_oid, u_tid))
def test_getObjectHistory(self): oid = p64(1) tid1, tid2, tid3 = self.getTIDs(3) txn1, objs1 = self.getTransaction([oid]) txn2, objs2 = self.getTransaction([oid]) txn3, objs3 = self.getTransaction([oid]) # one revision self.db.storeTransaction(tid1, objs1, txn1, False) result = self.db.getObjectHistory(oid, 0, 3) self.assertEqual(result, [(tid1, 0)]) result = self.db.getObjectHistory(oid, 1, 1) self.assertEqual(result, None) # two revisions self.db.storeTransaction(tid2, objs2, txn2, False) result = self.db.getObjectHistory(oid, 0, 3) self.assertEqual(result, [(tid2, 0), (tid1, 0)]) result = self.db.getObjectHistory(oid, 1, 3) self.assertEqual(result, [(tid1, 0)]) result = self.db.getObjectHistory(oid, 2, 3) self.assertEqual(result, None)
def getLastIDs(self): tid, _, _, oid = self.db.getLastIDs() return (max(tid, util.p64(self.zodb_ltid)), None, None, max(oid, util.p64(self.zodb_loid)))
def getLastIDs(self): tid, oid = self.db.getLastIDs() return (max(tid, util.p64(self.zodb_ltid)), max(oid, util.p64(self.zodb_loid)))
def asTID(self, value): if '.' in value: return tidFromTime(float(value)) return p64(int(value, 0))
def getTIDList(self, offset, length, partition_list): q = self.query r = q("""SELECT tid FROM trans WHERE `partition` in (%s) ORDER BY tid DESC LIMIT %d,%d""" \ % (','.join(map(str, partition_list)), offset, length)) return [util.p64(t[0]) for t in r]
def load(self, oid, tid=None, before_tid=None): """ Internal method which manage load, loadSerial and loadBefore. OID and TID (serial) parameters are expected packed. oid OID of object to get. tid If given, the exact serial at which OID is desired. before_tid should be None. before_tid If given, the excluded upper bound serial at which OID is desired. serial should be None. Return value: (3-tuple) - Object data (None if object creation was undone). - Serial of given data. - Next serial at which object exists, or None. Only set when tid parameter is not None. Exceptions: NEOStorageError technical problem NEOStorageNotFoundError object exists but no data satisfies given parameters NEOStorageDoesNotExistError object doesn't exist NEOStorageCreationUndoneError object existed, but its creation was undone Note that loadSerial is used during conflict resolution to load object's current version, which is not visible to us normaly (it was committed after our snapshot was taken). """ # TODO: # - rename parameters (here? and in handlers & packet definitions) # When not bound to a ZODB Connection, load() may be the # first method called and last_tid may still be None. # This happens, for example, when opening the DB. if not (tid or before_tid) and self.last_tid: # Do not get something more recent than the last invalidation # we got from master. before_tid = p64(u64(self.last_tid) + 1) acquire = self._cache_lock_acquire release = self._cache_lock_release # XXX: Consider using a more fine-grained lock. self._load_lock_acquire() try: acquire() try: result = self._loadFromCache(oid, tid, before_tid) if result: return result self._loading_oid = oid finally: release() data, tid, next_tid, _ = self._loadFromStorage(oid, tid, before_tid) acquire() try: if self._loading_oid: # Common case (no race condition). self._cache.store(oid, data, tid, next_tid) elif self._loading_invalidated: # oid has just been invalidated. if not next_tid: next_tid = self._loading_invalidated self._cache.store(oid, data, tid, next_tid) # Else, we just reconnected to the master. finally: release() finally: self._load_lock_release() return data, tid, next_tid
def undo(self, undone_tid, txn, tryToResolveConflict): txn_context = self._txn_container.get(txn) txn_info, txn_ext = self._getTransactionInformation(undone_tid) txn_oid_list = txn_info['oids'] # Regroup objects per partition, to ask a minimum set of storage. partition_oid_dict = {} for oid in txn_oid_list: partition = self.pt.getPartition(oid) try: oid_list = partition_oid_dict[partition] except KeyError: oid_list = partition_oid_dict[partition] = [] oid_list.append(oid) # Ask storage the undo serial (serial at which object's previous data # is) getCellList = self.pt.getCellList getCellSortKey = self.cp.getCellSortKey getConnForCell = self.cp.getConnForCell queue = self._thread_container.queue ttid = txn_context['ttid'] undo_object_tid_dict = {} snapshot_tid = p64(u64(self.last_tid) + 1) for partition, oid_list in partition_oid_dict.iteritems(): cell_list = getCellList(partition, readable=True) # We do want to shuffle before getting one with the smallest # key, so that all cells with the same (smallest) key has # identical chance to be chosen. shuffle(cell_list) storage_conn = getConnForCell(min(cell_list, key=getCellSortKey)) storage_conn.ask(Packets.AskObjectUndoSerial(ttid, snapshot_tid, undone_tid, oid_list), queue=queue, undo_object_tid_dict=undo_object_tid_dict) # Wait for all AnswerObjectUndoSerial. We might get OidNotFoundError, # meaning that objects in transaction's oid_list do not exist any # longer. This is the symptom of a pack, so forbid undoing transaction # when it happens. try: self.waitResponses(queue) except NEOStorageNotFoundError: self.dispatcher.forget_queue(queue) raise UndoError('non-undoable transaction') # Send undo data to all storage nodes. for oid in txn_oid_list: current_serial, undo_serial, is_current = undo_object_tid_dict[oid] if is_current: data = None else: # Serial being undone is not the latest version for this # object. This is an undo conflict, try to resolve it. try: # Load the latest version we are supposed to see data = self.load(oid, current_serial)[0] # Load the version we were undoing to undo_data = self.load(oid, undo_serial)[0] except NEOStorageNotFoundError: raise UndoError('Object not found while resolving undo ' 'conflict') # Resolve conflict try: data = tryToResolveConflict(oid, current_serial, undone_tid, undo_data, data) except ConflictError: raise UndoError('Some data were modified by a later ' \ 'transaction', oid) undo_serial = None self._store(txn_context, oid, current_serial, data, undo_serial) return None, txn_oid_list
def test_updateObjectDataForPack(self): ram_serial = self.getNextTID() oid = p64(1) orig_serial = self.getNextTID() uuid = self.getClientUUID() locking_serial = self.getNextTID() other_serial = self.getNextTID() new_serial = self.getNextTID() checksum = "2" * 20 self.register(uuid, locking_serial) # Object not known, nothing happens self.assertEqual( self.manager.getObjectFromTransaction(locking_serial, oid), None) self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum) self.assertEqual( self.manager.getObjectFromTransaction(locking_serial, oid), None) self.manager.abort(locking_serial, even_if_locked=True) # Object known, but doesn't point at orig_serial, it is not updated self.register(uuid, locking_serial) self.manager.storeObject(locking_serial, ram_serial, oid, 0, "3" * 20, 'bar', None) holdData = self.app.dm.mockGetNamedCalls('holdData') self.assertEqual(holdData.pop(0).params, ("3" * 20, oid, 'bar', 0)) orig_object = self.manager.getObjectFromTransaction( locking_serial, oid) self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum) self.assertEqual( self.manager.getObjectFromTransaction(locking_serial, oid), orig_object) self.manager.abort(locking_serial, even_if_locked=True) self.register(uuid, locking_serial) self.manager.storeObject(locking_serial, ram_serial, oid, None, None, None, other_serial) orig_object = self.manager.getObjectFromTransaction( locking_serial, oid) self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum) self.assertEqual( self.manager.getObjectFromTransaction(locking_serial, oid), orig_object) self.manager.abort(locking_serial, even_if_locked=True) # Object known and points at undone data it gets updated self.register(uuid, locking_serial) self.manager.storeObject(locking_serial, ram_serial, oid, None, None, None, orig_serial) self.manager.updateObjectDataForPack(oid, orig_serial, new_serial, checksum) self.assertEqual( self.manager.getObjectFromTransaction(locking_serial, oid), (oid, None, new_serial)) self.manager.abort(locking_serial, even_if_locked=True) self.register(uuid, locking_serial) self.manager.storeObject(locking_serial, ram_serial, oid, None, None, None, orig_serial) self.manager.updateObjectDataForPack(oid, orig_serial, None, checksum) self.assertEqual(holdData.pop(0).params, (checksum, )) self.assertEqual( self.manager.getObjectFromTransaction(locking_serial, oid), (oid, checksum, None)) self.manager.abort(locking_serial, even_if_locked=True) self.assertFalse(holdData)
def undo(self, undone_tid, txn, tryToResolveConflict): txn_context = self._txn_container.get(txn) txn_info, txn_ext = self._getTransactionInformation(undone_tid) txn_oid_list = txn_info['oids'] # Regroup objects per partition, to ask a minimum set of storage. partition_oid_dict = {} for oid in txn_oid_list: partition = self.pt.getPartition(oid) try: oid_list = partition_oid_dict[partition] except KeyError: oid_list = partition_oid_dict[partition] = [] oid_list.append(oid) # Ask storage the undo serial (serial at which object's previous data # is) getCellList = self.pt.getCellList getCellSortKey = self.cp.getCellSortKey getConnForCell = self.cp.getConnForCell queue = self._thread_container.queue ttid = txn_context['ttid'] undo_object_tid_dict = {} snapshot_tid = p64(u64(self.last_tid) + 1) for partition, oid_list in partition_oid_dict.iteritems(): cell_list = getCellList(partition, readable=True) # We do want to shuffle before getting one with the smallest # key, so that all cells with the same (smallest) key has # identical chance to be chosen. shuffle(cell_list) storage_conn = getConnForCell(min(cell_list, key=getCellSortKey)) storage_conn.ask(Packets.AskObjectUndoSerial( ttid, snapshot_tid, undone_tid, oid_list), queue=queue, undo_object_tid_dict=undo_object_tid_dict) # Wait for all AnswerObjectUndoSerial. We might get OidNotFoundError, # meaning that objects in transaction's oid_list do not exist any # longer. This is the symptom of a pack, so forbid undoing transaction # when it happens. try: self.waitResponses(queue) except NEOStorageNotFoundError: self.dispatcher.forget_queue(queue) raise UndoError('non-undoable transaction') # Send undo data to all storage nodes. for oid in txn_oid_list: current_serial, undo_serial, is_current = undo_object_tid_dict[oid] if is_current: data = None else: # Serial being undone is not the latest version for this # object. This is an undo conflict, try to resolve it. try: # Load the latest version we are supposed to see data = self.load(oid, current_serial)[0] # Load the version we were undoing to undo_data = self.load(oid, undo_serial)[0] except NEOStorageNotFoundError: raise UndoError('Object not found while resolving undo ' 'conflict') # Resolve conflict try: data = tryToResolveConflict(oid, current_serial, undone_tid, undo_data, data) except ConflictError: raise UndoError('Some data were modified by a later ' \ 'transaction', oid) undo_serial = None self._store(txn_context, oid, current_serial, data, undo_serial) return None, txn_oid_list