class test_cursor_comparison(wttest.WiredTigerTestCase): name = 'test_compare' types = [('file', dict(type='file:', lsm=False, dataset=SimpleDataSet)), ('lsm', dict(type='table:', lsm=True, dataset=ComplexLSMDataSet)), ('table', dict(type='table:', lsm=False, dataset=ComplexDataSet))] keyfmt = [('integer', dict(keyfmt='i')), ('recno', dict(keyfmt='r')), ('string', dict(keyfmt='S'))] # Skip record number keys with LSM. scenarios = filter_scenarios( make_scenarios(types, keyfmt), lambda name, d: not (d['lsm'] and d['keyfmt'] == 'r')) def test_cursor_comparison(self): uri = self.type + 'compare' uriX = self.type + 'compareX' # Build the object. ds = self.dataset(self, uri, 100, key_format=self.keyfmt) dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt) ds.populate() dsX.populate() if self.type == 'file:': ix0_0 = None ix0_1 = None ix1_0 = None ixX_0 = None else: ix0_0 = self.session.open_cursor(ds.index_name(0), None) ix0_1 = self.session.open_cursor(ds.index_name(0), None) ix1_0 = self.session.open_cursor(ds.index_name(1), None) ixX_0 = self.session.open_cursor(dsX.index_name(0), None) ix0_0.next() ix0_1.next() ix1_0.next() ixX_0.next() c1 = self.session.open_cursor(uri, None) c2 = self.session.open_cursor(uri, None) # Confirm failure unless the keys are set. msg = '/requires key be set/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c1.compare(c2), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c2.compare(c1), msg) # Test cursors before they're positioned. c1.set_key(ds.key(10)) c2.set_key(ds.key(20)) self.assertGreater(c2.compare(c1), 0) self.assertLess(c1.compare(c2), 0) c2.set_key(ds.key(10)) self.assertEqual(c1.compare(c2), 0) self.assertEqual(c2.compare(c1), 0) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) cX.set_key(dsX.key(10)) msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cX.compare(c1), msg) msg = '/wt_cursor.* is None/' self.assertRaisesHavingMessage(RuntimeError, lambda: cX.compare(None), msg) if ix0_0 != None: self.assertEqual(ix0_0.compare(ix0_1), 0) ix0_1.reset() ix0_1.prev() self.assertLess(ix0_0.compare(ix0_1), 0) self.assertGreater(ix0_1.compare(ix0_0), 0) # Main table vs. index not allowed msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c1.compare(ix0_0), msg) # Two unrelated indices not allowed self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: ixX_0.compare(ix0_0), msg) # Two different indices from same table not allowed self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: ix0_0.compare(ix1_0), msg) # Test cursors after they're positioned (shouldn't matter for compare). c1.set_key(ds.key(10)) self.assertEqual(c1.search(), 0) c2.set_key(ds.key(20)) self.assertEqual(c2.search(), 0) self.assertGreater(c2.compare(c1), 0) self.assertLess(c1.compare(c2), 0) c2.set_key(ds.key(10)) self.assertEqual(c2.search(), 0) self.assertEqual(c1.compare(c2), 0) self.assertEqual(c2.compare(c1), 0) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) cX.set_key(dsX.key(10)) self.assertEqual(cX.search(), 0) msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cX.compare(c1), msg) def test_cursor_equality(self): uri = self.type + 'equality' uriX = self.type + 'compareX' # Build the object. ds = self.dataset(self, uri, 100, key_format=self.keyfmt) dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt) ds.populate() dsX.populate() if self.type == 'file:': ix0_0 = None ix0_1 = None ix1_0 = None ixX_0 = None else: ix0_0 = self.session.open_cursor(ds.index_name(0), None) ix0_1 = self.session.open_cursor(ds.index_name(0), None) ix1_0 = self.session.open_cursor(ds.index_name(1), None) ixX_0 = self.session.open_cursor(dsX.index_name(0), None) ix0_0.next() ix0_1.next() ix1_0.next() ixX_0.next() c1 = self.session.open_cursor(uri, None) c2 = self.session.open_cursor(uri, None) # Confirm failure unless the keys are set. msg = '/requires key be set/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c1.equals(c2), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c2.equals(c1), msg) # Test cursors before they're positioned. c1.set_key(ds.key(10)) c2.set_key(ds.key(20)) self.assertFalse(c1.equals(c2)) self.assertFalse(c2.equals(c1)) c2.set_key(ds.key(10)) self.assertTrue(c1.equals(c2)) self.assertTrue(c2.equals(c1)) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) cX.set_key(dsX.key(10)) msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cX.equals(c1), msg) msg = '/wt_cursor.* is None/' self.assertRaisesHavingMessage(RuntimeError, lambda: cX.equals(None), msg) if ix0_0 != None: self.assertTrue(ix0_0.equals(ix0_1)) ix0_1.reset() ix0_1.prev() self.assertFalse(ix0_0.equals(ix0_1)) # Main table vs. index not allowed msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c1.equals(ix0_0), msg) # Two unrelated indices not allowed self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: ixX_0.equals(ix0_0), msg) # Two different indices from same table not allowed self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: ix0_0.equals(ix1_0), msg) # Test cursors after they're positioned (internally, it's a different # search path if keys are positioned in the tree). c1.set_key(ds.key(10)) self.assertEqual(c1.search(), 0) c2.set_key(ds.key(20)) self.assertEqual(c2.search(), 0) self.assertFalse(c1.equals(c2)) self.assertFalse(c2.equals(c1)) c2.set_key(ds.key(10)) self.assertEqual(c2.search(), 0) self.assertTrue(c1.equals(c2)) self.assertTrue(c2.equals(c1)) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) cX.set_key(dsX.key(10)) self.assertEqual(cX.search(), 0) msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cX.equals(c1), msg)
class test_cursor12(wttest.WiredTigerTestCase): keyfmt = [ ('recno', dict(keyfmt='r')), ('string', dict(keyfmt='S')), ] valuefmt = [ ('item', dict(valuefmt='u')), ('string', dict(valuefmt='S')), ] types = [ ('file', dict(uri='file:modify')), ('lsm', dict(uri='lsm:modify')), ('table', dict(uri='table:modify')), ] # Skip record number keys with LSM. scenarios = filter_scenarios(make_scenarios(types, keyfmt, valuefmt), lambda name, d: not ('lsm' in d['uri'] and d['keyfmt'] == 'r')) # List with original value, final value, and modifications to get # there. list = [ { 'o' : 'ABCDEFGH', # no operation 'f' : 'ABCDEFGH', 'mods' : [['', 0, 0]] },{ 'o' : 'ABCDEFGH', # no operation with offset 'f' : 'ABCDEFGH', 'mods' : [['', 4, 0]] },{ 'o' : 'ABCDEFGH', # rewrite beginning 'f' : '--CDEFGH', 'mods' : [['--', 0, 2]] },{ 'o' : 'ABCDEFGH', # rewrite end 'f' : 'ABCDEF--', 'mods' : [['--', 6, 2]] },{ 'o' : 'ABCDEFGH', # append 'f' : 'ABCDEFGH--', 'mods' : [['--', 8, 2]] },{ 'o' : 'ABCDEFGH', # append with gap 'f' : 'ABCDEFGH --', 'mods' : [['--', 10, 2]] },{ 'o' : 'ABCDEFGH', # multiple replacements 'f' : 'A-C-E-G-', 'mods' : [['-', 1, 1], ['-', 3, 1], ['-', 5, 1], ['-', 7, 1]] },{ 'o' : 'ABCDEFGH', # multiple overlapping replacements 'f' : 'A-CDEFGH', 'mods' : [['+', 1, 1], ['+', 1, 1], ['+', 1, 1], ['-', 1, 1]] },{ 'o' : 'ABCDEFGH', # multiple overlapping gap replacements 'f' : 'ABCDEFGH --', 'mods' : [['+', 10, 1], ['+', 10, 1], ['+', 10, 1], ['--', 10, 2]] },{ 'o' : 'ABCDEFGH', # shrink beginning 'f' : '--EFGH', 'mods' : [['--', 0, 4]] },{ 'o' : 'ABCDEFGH', # shrink middle 'f' : 'AB--GH', 'mods' : [['--', 2, 4]] },{ 'o' : 'ABCDEFGH', # shrink end 'f' : 'ABCD--', 'mods' : [['--', 4, 4]] },{ 'o' : 'ABCDEFGH', # grow beginning 'f' : '--ABCDEFGH', 'mods' : [['--', 0, 0]] },{ 'o' : 'ABCDEFGH', # grow middle 'f' : 'ABCD--EFGH', 'mods' : [['--', 4, 0]] },{ 'o' : 'ABCDEFGH', # grow end 'f' : 'ABCDEFGH--', 'mods' : [['--', 8, 0]] },{ 'o' : 'ABCDEFGH', # discard beginning 'f' : 'EFGH', 'mods' : [['', 0, 4]] },{ 'o' : 'ABCDEFGH', # discard middle 'f' : 'ABGH', 'mods' : [['', 2, 4]] },{ 'o' : 'ABCDEFGH', # discard end 'f' : 'ABCD', 'mods' : [['', 4, 4]] },{ 'o' : 'ABCDEFGH', # discard everything 'f' : '', 'mods' : [['', 0, 8]] },{ 'o' : 'ABCDEFGH', # overlap the end and append 'f' : 'ABCDEF--XX', 'mods' : [['--XX', 6, 2]] },{ 'o' : 'ABCDEFGH', # overlap the end with incorrect size 'f' : 'ABCDEFG01234567', 'mods' : [['01234567', 7, 2000]] },{ # many updates 'o' : '-ABCDEFGHIJKLMNOPQRSTUVWXYZ-', 'f' : '-eeeeeeeeeeeeeeeeeeeeeeeeee-', 'mods' : [['a', 1, 1], ['a', 2, 1], ['a', 3, 1], ['a', 4, 1], ['a', 5, 1], ['a', 6, 1], ['a', 7, 1], ['a', 8, 1], ['a', 9, 1], ['a', 10, 1], ['a', 11, 1], ['a', 12, 1], ['a', 13, 1], ['a', 14, 1], ['a', 15, 1], ['a', 16, 1], ['a', 17, 1], ['a', 18, 1], ['a', 19, 1], ['a', 20, 1], ['a', 21, 1], ['a', 22, 1], ['a', 23, 1], ['a', 24, 1], ['a', 25, 1], ['a', 26, 1], ['b', 1, 1], ['b', 2, 1], ['b', 3, 1], ['b', 4, 1], ['b', 5, 1], ['b', 6, 1], ['b', 7, 1], ['b', 8, 1], ['b', 9, 1], ['b', 10, 1], ['b', 11, 1], ['b', 12, 1], ['b', 13, 1], ['b', 14, 1], ['b', 15, 1], ['b', 16, 1], ['b', 17, 1], ['b', 18, 1], ['b', 19, 1], ['b', 20, 1], ['b', 21, 1], ['b', 22, 1], ['b', 23, 1], ['b', 24, 1], ['b', 25, 1], ['b', 26, 1], ['c', 1, 1], ['c', 2, 1], ['c', 3, 1], ['c', 4, 1], ['c', 5, 1], ['c', 6, 1], ['c', 7, 1], ['c', 8, 1], ['c', 9, 1], ['c', 10, 1], ['c', 11, 1], ['c', 12, 1], ['c', 13, 1], ['c', 14, 1], ['c', 15, 1], ['c', 16, 1], ['c', 17, 1], ['c', 18, 1], ['c', 19, 1], ['c', 20, 1], ['c', 21, 1], ['c', 22, 1], ['c', 23, 1], ['c', 24, 1], ['c', 25, 1], ['c', 26, 1], ['d', 1, 1], ['d', 2, 1], ['d', 3, 1], ['d', 4, 1], ['d', 5, 1], ['d', 6, 1], ['d', 7, 1], ['d', 8, 1], ['d', 9, 1], ['d', 10, 1], ['d', 11, 1], ['d', 12, 1], ['d', 13, 1], ['d', 14, 1], ['d', 15, 1], ['d', 16, 1], ['d', 17, 1], ['d', 18, 1], ['d', 19, 1], ['d', 20, 1], ['d', 21, 1], ['d', 22, 1], ['d', 23, 1], ['d', 24, 1], ['d', 25, 1], ['d', 26, 1], ['e', 1, 1], ['e', 2, 1], ['e', 3, 1], ['e', 4, 1], ['e', 5, 1], ['e', 6, 1], ['e', 7, 1], ['e', 8, 1], ['e', 9, 1], ['e', 10, 1], ['e', 11, 1], ['e', 12, 1], ['e', 13, 1], ['e', 14, 1], ['e', 15, 1], ['e', 16, 1], ['e', 17, 1], ['e', 18, 1], ['e', 19, 1], ['e', 20, 1], ['e', 21, 1], ['e', 22, 1], ['e', 23, 1], ['e', 24, 1], ['e', 25, 1], ['e', 26, 1]] } ] # Create a set of modified records and verify in-memory reads. def modify_load(self, ds, single): # For each test in the list: # set the original value, # apply modifications in order, # confirm the final state row = 10 c = self.session.open_cursor(self.uri, None) for i in self.list: c.set_key(ds.key(row)) c.set_value(i['o']) self.assertEquals(c.update(), 0) c.reset() c.set_key(ds.key(row)) mods = [] for j in i['mods']: mod = wiredtiger.Modify(j[0], j[1], j[2]) mods.append(mod) self.assertEquals(c.modify(mods), 0) c.reset() c.set_key(ds.key(row)) self.assertEquals(c.search(), 0) v = c.get_value() self.assertEquals(v.replace("\x00", " "), i['f']) if not single: row = row + 1 c.close() # Confirm the modified records are correct. def modify_confirm(self, ds, single): # For each test in the list: # confirm the final state is there. row = 10 c = self.session.open_cursor(self.uri, None) for i in self.list: c.set_key(ds.key(row)) self.assertEquals(c.search(), 0) v = c.get_value() self.assertEquals(v.replace("\x00", " "), i['f']) if not single: row = row + 1 c.close() # Smoke-test the modify API, operating on a group of records. def test_modify_smoke(self): ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() self.modify_load(ds, False) # Smoke-test the modify API, operating on a single record def test_modify_smoke_single(self): ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() self.modify_load(ds, True) # Smoke-test the modify API, closing and re-opening the database. def test_modify_smoke_reopen(self): ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() self.modify_load(ds, False) # Flush to disk, forcing reconciliation. self.reopen_conn() self.modify_confirm(ds, False) # Smoke-test the modify API, recovering the database. def test_modify_smoke_recover(self): # Close the original database. self.conn.close() # Open a new database with logging configured. self.conn_config = \ 'log=(enabled=true),transaction_sync=(method=dsync,enabled)' self.conn = self.setUpConnectionOpen(".") self.session = self.setUpSessionOpen(self.conn) # Populate a database, and checkpoint it so it exists after recovery. ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() self.session.checkpoint() self.modify_load(ds, False) # Crash and recover in a new directory. newdir = 'RESTART' copy_wiredtiger_home('.', newdir) self.conn.close() self.conn = self.setUpConnectionOpen(newdir) self.session = self.setUpSessionOpen(self.conn) self.session.verify(self.uri) self.modify_confirm(ds, False) # Check that we can perform a large number of modifications to a record. def test_modify_many(self): ds = SimpleDataSet(self, self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() c = self.session.open_cursor(self.uri, None) c.set_key(ds.key(10)) orig = 'abcdefghijklmnopqrstuvwxyz' c.set_value(orig) self.assertEquals(c.update(), 0) for i in range(0, 50000): new = "".join([random.choice(string.digits) for i in xrange(5)]) orig = orig[:10] + new + orig[15:] mods = [] mod = wiredtiger.Modify(new, 10, 5) mods.append(mod) self.assertEquals(c.modify(mods), 0) c.set_key(ds.key(10)) self.assertEquals(c.search(), 0) self.assertEquals(c.get_value(), orig) # Check that modify returns not-found after a delete. def test_modify_delete(self): ds = SimpleDataSet(self, self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() c = self.session.open_cursor(self.uri, None) c.set_key(ds.key(10)) self.assertEquals(c.remove(), 0) mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(10)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) # Check that modify returns not-found when an insert is not yet committed # and after it's aborted. def test_modify_abort(self): ds = SimpleDataSet(self, self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() # Start a transaction. self.session.begin_transaction() # Insert a new record. c = self.session.open_cursor(self.uri, None) c.set_key(ds.key(30)) c.set_value(ds.value(30)) self.assertEquals(c.insert(), 0) # Test that we can successfully modify our own record. mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(30)) self.assertEqual(c.modify(mods), 0) # Test that another transaction cannot modify our uncommitted record. xs = self.conn.open_session() xc = xs.open_cursor(self.uri, None) xc.set_key(ds.key(30)) xc.set_value(ds.value(30)) mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) xc.set_key(ds.key(30)) self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND) # Rollback our transaction. self.session.rollback_transaction() # Test that we can't modify our aborted insert. mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(30)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
class test_cursor12(wttest.WiredTigerTestCase): keyfmt = [ ('recno', dict(keyfmt='r')), ('string', dict(keyfmt='S')), ] valuefmt = [ ('item', dict(valuefmt='u')), ('string', dict(valuefmt='S')), ] types = [ ('file', dict(uri='file:modify')), ('lsm', dict(uri='lsm:modify')), ('table', dict(uri='table:modify')), ] # Skip record number keys with LSM. scenarios = filter_scenarios(make_scenarios(types, keyfmt, valuefmt), lambda name, d: not ('lsm' in d['uri'] and d['keyfmt'] == 'r')) # List with original value, final value, and modifications to get # there. list = [ { 'o' : 'ABCDEFGH', # no operation 'f' : 'ABCDEFGH', 'mods' : [['', 0, 0]] },{ 'o' : 'ABCDEFGH', # no operation with offset 'f' : 'ABCDEFGH', 'mods' : [['', 4, 0]] },{ 'o' : 'ABCDEFGH', # rewrite beginning 'f' : '--CDEFGH', 'mods' : [['--', 0, 2]] },{ 'o' : 'ABCDEFGH', # rewrite end 'f' : 'ABCDEF--', 'mods' : [['--', 6, 2]] },{ 'o' : 'ABCDEFGH', # append 'f' : 'ABCDEFGH--', 'mods' : [['--', 8, 2]] },{ 'o' : 'ABCDEFGH', # append with gap 'f' : 'ABCDEFGH --', 'mods' : [['--', 10, 2]] },{ 'o' : 'ABCDEFGH', # multiple replacements 'f' : 'A-C-E-G-', 'mods' : [['-', 1, 1], ['-', 3, 1], ['-', 5, 1], ['-', 7, 1]] },{ 'o' : 'ABCDEFGH', # multiple overlapping replacements 'f' : 'A-CDEFGH', 'mods' : [['+', 1, 1], ['+', 1, 1], ['+', 1, 1], ['-', 1, 1]] },{ 'o' : 'ABCDEFGH', # multiple overlapping gap replacements 'f' : 'ABCDEFGH --', 'mods' : [['+', 10, 1], ['+', 10, 1], ['+', 10, 1], ['--', 10, 2]] },{ 'o' : 'ABCDEFGH', # shrink beginning 'f' : '--EFGH', 'mods' : [['--', 0, 4]] },{ 'o' : 'ABCDEFGH', # shrink middle 'f' : 'AB--GH', 'mods' : [['--', 2, 4]] },{ 'o' : 'ABCDEFGH', # shrink end 'f' : 'ABCD--', 'mods' : [['--', 4, 4]] },{ 'o' : 'ABCDEFGH', # grow beginning 'f' : '--ABCDEFGH', 'mods' : [['--', 0, 0]] },{ 'o' : 'ABCDEFGH', # grow middle 'f' : 'ABCD--EFGH', 'mods' : [['--', 4, 0]] },{ 'o' : 'ABCDEFGH', # grow end 'f' : 'ABCDEFGH--', 'mods' : [['--', 8, 0]] },{ 'o' : 'ABCDEFGH', # discard beginning 'f' : 'EFGH', 'mods' : [['', 0, 4]] },{ 'o' : 'ABCDEFGH', # discard middle 'f' : 'ABGH', 'mods' : [['', 2, 4]] },{ 'o' : 'ABCDEFGH', # discard end 'f' : 'ABCD', 'mods' : [['', 4, 4]] },{ 'o' : 'ABCDEFGH', # discard everything 'f' : '', 'mods' : [['', 0, 8]] },{ 'o' : 'ABCDEFGH', # overlap the end and append 'f' : 'ABCDEF--XX', 'mods' : [['--XX', 6, 2]] },{ 'o' : 'ABCDEFGH', # overlap the end with incorrect size 'f' : 'ABCDEFG01234567', 'mods' : [['01234567', 7, 2000]] },{ # many updates 'o' : '-ABCDEFGHIJKLMNOPQRSTUVWXYZ-', 'f' : '-eeeeeeeeeeeeeeeeeeeeeeeeee-', 'mods' : [['a', 1, 1], ['a', 2, 1], ['a', 3, 1], ['a', 4, 1], ['a', 5, 1], ['a', 6, 1], ['a', 7, 1], ['a', 8, 1], ['a', 9, 1], ['a', 10, 1], ['a', 11, 1], ['a', 12, 1], ['a', 13, 1], ['a', 14, 1], ['a', 15, 1], ['a', 16, 1], ['a', 17, 1], ['a', 18, 1], ['a', 19, 1], ['a', 20, 1], ['a', 21, 1], ['a', 22, 1], ['a', 23, 1], ['a', 24, 1], ['a', 25, 1], ['a', 26, 1], ['b', 1, 1], ['b', 2, 1], ['b', 3, 1], ['b', 4, 1], ['b', 5, 1], ['b', 6, 1], ['b', 7, 1], ['b', 8, 1], ['b', 9, 1], ['b', 10, 1], ['b', 11, 1], ['b', 12, 1], ['b', 13, 1], ['b', 14, 1], ['b', 15, 1], ['b', 16, 1], ['b', 17, 1], ['b', 18, 1], ['b', 19, 1], ['b', 20, 1], ['b', 21, 1], ['b', 22, 1], ['b', 23, 1], ['b', 24, 1], ['b', 25, 1], ['b', 26, 1], ['c', 1, 1], ['c', 2, 1], ['c', 3, 1], ['c', 4, 1], ['c', 5, 1], ['c', 6, 1], ['c', 7, 1], ['c', 8, 1], ['c', 9, 1], ['c', 10, 1], ['c', 11, 1], ['c', 12, 1], ['c', 13, 1], ['c', 14, 1], ['c', 15, 1], ['c', 16, 1], ['c', 17, 1], ['c', 18, 1], ['c', 19, 1], ['c', 20, 1], ['c', 21, 1], ['c', 22, 1], ['c', 23, 1], ['c', 24, 1], ['c', 25, 1], ['c', 26, 1], ['d', 1, 1], ['d', 2, 1], ['d', 3, 1], ['d', 4, 1], ['d', 5, 1], ['d', 6, 1], ['d', 7, 1], ['d', 8, 1], ['d', 9, 1], ['d', 10, 1], ['d', 11, 1], ['d', 12, 1], ['d', 13, 1], ['d', 14, 1], ['d', 15, 1], ['d', 16, 1], ['d', 17, 1], ['d', 18, 1], ['d', 19, 1], ['d', 20, 1], ['d', 21, 1], ['d', 22, 1], ['d', 23, 1], ['d', 24, 1], ['d', 25, 1], ['d', 26, 1], ['e', 1, 1], ['e', 2, 1], ['e', 3, 1], ['e', 4, 1], ['e', 5, 1], ['e', 6, 1], ['e', 7, 1], ['e', 8, 1], ['e', 9, 1], ['e', 10, 1], ['e', 11, 1], ['e', 12, 1], ['e', 13, 1], ['e', 14, 1], ['e', 15, 1], ['e', 16, 1], ['e', 17, 1], ['e', 18, 1], ['e', 19, 1], ['e', 20, 1], ['e', 21, 1], ['e', 22, 1], ['e', 23, 1], ['e', 24, 1], ['e', 25, 1], ['e', 26, 1]] } ] def setUp(self): if sys.version_info[0] >= 3 and self.valuefmt == 'u': # Python3 distinguishes bytes from strings self.nullbyte = b'\x00' self.spacebyte = b' ' else: self.nullbyte = '\x00' self.spacebyte = ' ' super(test_cursor12, self).setUp() # Convert a string to the correct type for the value. def make_value(self, s): if self.valuefmt == 'u': return bytes(s.encode()) else: return s def fix_mods(self, mods): if bytes != str and self.valuefmt == 'u': # In Python3, bytes and strings are independent types, and # the WiredTiger API needs bytes when the format calls for bytes. newmods = [] for mod in mods: # We need to check because we may converted some of the Modify # records already. if type(mod.data) == str: newmods.append(wiredtiger.Modify( self.make_value(mod.data), mod.offset, mod.size)) else: newmods.append(mod) mods = newmods return mods # Create a set of modified records and verify in-memory reads. def modify_load(self, ds, single): # For each test in the list: # set the original value, # apply modifications in order, # confirm the final state row = 10 c = self.session.open_cursor(self.uri, None) for i in self.list: c.set_key(ds.key(row)) c.set_value(self.make_value(i['o'])) self.assertEquals(c.update(), 0) c.reset() self.session.begin_transaction("isolation=snapshot") c.set_key(ds.key(row)) mods = [] for j in i['mods']: mod = wiredtiger.Modify(j[0], j[1], j[2]) mods.append(mod) mods = self.fix_mods(mods) self.assertEquals(c.modify(mods), 0) self.session.commit_transaction() c.reset() c.set_key(ds.key(row)) self.assertEquals(c.search(), 0) v = c.get_value() expect = self.make_value(i['f']) self.assertEquals(v.replace(self.nullbyte, self.spacebyte), expect) if not single: row = row + 1 c.close() # Confirm the modified records are correct. def modify_confirm(self, ds, single): # For each test in the list: # confirm the final state is there. row = 10 c = self.session.open_cursor(self.uri, None) for i in self.list: c.set_key(ds.key(row)) self.assertEquals(c.search(), 0) v = c.get_value() expect = self.make_value(i['f']) self.assertEquals(v.replace(self.nullbyte, self.spacebyte), expect) if not single: row = row + 1 c.close() # Smoke-test the modify API, anything other than an explicit transaction # in snapshot isolation fails. def test_modify_txn_api(self): ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() c = self.session.open_cursor(self.uri, None) c.set_key(ds.key(10)) msg = '/not supported/' self.session.begin_transaction() mods = [] mods.append(wiredtiger.Modify('-', 1, 1)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c.modify(mods), msg) self.session.rollback_transaction() self.session.begin_transaction("isolation=read-uncommitted") mods = [] mods.append(wiredtiger.Modify('-', 1, 1)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c.modify(mods), msg) self.session.rollback_transaction() self.session.begin_transaction("isolation=read-committed") mods = [] mods.append(wiredtiger.Modify('-', 1, 1)) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c.modify(mods), msg) self.session.rollback_transaction() # Smoke-test the modify API, operating on a group of records. def test_modify_smoke(self): ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() self.modify_load(ds, False) # Smoke-test the modify API, operating on a single record def test_modify_smoke_single(self): ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() self.modify_load(ds, True) # Smoke-test the modify API, closing and re-opening the database. def test_modify_smoke_reopen(self): ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() self.modify_load(ds, False) # Flush to disk, forcing reconciliation. self.reopen_conn() self.modify_confirm(ds, False) # Smoke-test the modify API, recovering the database. def test_modify_smoke_recover(self): # Close the original database. self.conn.close() # Open a new database with logging configured. self.conn_config = \ 'log=(enabled=true),transaction_sync=(method=dsync,enabled)' self.conn = self.setUpConnectionOpen(".") self.session = self.setUpSessionOpen(self.conn) # Populate a database, and checkpoint it so it exists after recovery. ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() self.session.checkpoint() self.modify_load(ds, False) # Crash and recover in a new directory. newdir = 'RESTART' copy_wiredtiger_home('.', newdir) self.conn.close() self.conn = self.setUpConnectionOpen(newdir) self.session = self.setUpSessionOpen(self.conn) self.session.verify(self.uri) self.modify_confirm(ds, False) # Check that we can perform a large number of modifications to a record. def test_modify_many(self): ds = SimpleDataSet(self, self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() c = self.session.open_cursor(self.uri, None) self.session.begin_transaction("isolation=snapshot") c.set_key(ds.key(10)) orig = self.make_value('abcdefghijklmnopqrstuvwxyz') c.set_value(orig) self.assertEquals(c.update(), 0) for i in range(0, 50000): new = self.make_value("".join([random.choice(string.digits) \ for i in range(5)])) orig = orig[:10] + new + orig[15:] mods = [] mod = wiredtiger.Modify(new, 10, 5) mods.append(mod) mods = self.fix_mods(mods) self.assertEquals(c.modify(mods), 0) self.session.commit_transaction() c.set_key(ds.key(10)) self.assertEquals(c.search(), 0) self.assertEquals(c.get_value(), orig) # Check that modify returns not-found after a delete. def test_modify_delete(self): ds = SimpleDataSet(self, self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() c = self.session.open_cursor(self.uri, None) c.set_key(ds.key(10)) self.assertEquals(c.remove(), 0) self.session.begin_transaction("isolation=snapshot") mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) mods = self.fix_mods(mods) c.set_key(ds.key(10)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) self.session.commit_transaction() # Check that modify returns not-found when an insert is not yet committed # and after it's aborted. def test_modify_abort(self): ds = SimpleDataSet(self, self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() # Start a transaction. self.session.begin_transaction("isolation=snapshot") # Insert a new record. c = self.session.open_cursor(self.uri, None) c.set_key(ds.key(30)) c.set_value(ds.value(30)) self.assertEquals(c.insert(), 0) # Test that we can successfully modify our own record. mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(30)) mods = self.fix_mods(mods) self.assertEqual(c.modify(mods), 0) # Test that another transaction cannot modify our uncommitted record. xs = self.conn.open_session() xc = xs.open_cursor(self.uri, None) xs.begin_transaction("isolation=snapshot") xc.set_key(ds.key(30)) xc.set_value(ds.value(30)) mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) mods = self.fix_mods(mods) xc.set_key(ds.key(30)) self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND) xs.rollback_transaction() # Rollback our transaction. self.session.rollback_transaction() # Test that we can't modify our aborted insert. self.session.begin_transaction("isolation=snapshot") mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) mods = self.fix_mods(mods) c.set_key(ds.key(30)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction()
class test_overwrite(wttest.WiredTigerTestCase): name = 'overwrite' keyfmt = [ ('integer', dict(keyfmt='i')), ('recno', dict(keyfmt='r')), ('string', dict(keyfmt='S')), ] types = [ ('file', dict(uri='file:', lsm=False, ds=SimpleDataSet)), ('lsm', dict(uri='lsm:', lsm=True, ds=SimpleDataSet)), ('table-complex', dict(uri='table:', lsm=False, ds=ComplexDataSet)), ('table-complex-lsm', dict(uri='table:', lsm=True, ds=ComplexLSMDataSet)), ('table-index', dict(uri='table:', lsm=False, ds=SimpleIndexDataSet)), ('table-simple', dict(uri='table:', lsm=False, ds=SimpleDataSet)), ('table-simple-lsm', dict(uri='table:', lsm=True, ds=SimpleLSMDataSet)), ] # Skip record number keys with LSM. scenarios = filter_scenarios( make_scenarios(types, keyfmt), lambda name, d: not (d['lsm'] and d['keyfmt'] == 'r')) # Confirm a cursor configured with/without overwrite correctly handles # non-existent records during insert, remove and update operations. def test_overwrite_insert(self): uri = self.uri + self.name ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Insert of an existing record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) cursor.set_value(ds.value(1000)) self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) # One additional test for the insert method: duplicate the cursor with # overwrite configured and then the insert should succeed. This test # is only for the insert method because the remove and update method # failure modes are for non-existent records, and you cannot duplicate # cursor pointing to non-existent records. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) dupc = self.session.open_cursor(None, cursor, "overwrite=true") dupc.set_value(ds.value(1001)) self.assertEquals(dupc.insert(), 0) # Insert of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(6)) cursor.set_value(ds.value(1002)) self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) cursor.set_value(ds.value(1003)) self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(201)) cursor.set_value(ds.value(1004)) self.assertEquals(cursor.insert(), 0) def test_overwrite_remove(self): uri = self.uri + self.name ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Remove of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) self.assertEquals(cursor.remove(), 0) # Remove of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(6)) self.assertEquals(cursor.remove(), 0) # Remove of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) self.assertEquals(cursor.remove(), wiredtiger.WT_NOTFOUND) # Remove of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(201)) self.assertEquals(cursor.remove(), 0) def test_overwrite_update(self): uri = self.uri + self.name ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Update of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) cursor.set_value(ds.value(1005)) self.assertEquals(cursor.update(), 0) # Update of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(6)) cursor.set_value(ds.value(1006)) self.assertEquals(cursor.update(), 0) # Update of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) cursor.set_value(ds.value(1007)) self.assertEquals(cursor.update(), wiredtiger.WT_NOTFOUND) # Update of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(201)) cursor.set_value(ds.value(1008)) self.assertEquals(cursor.update(), 0)
class test_rollback_to_stable25(wttest.WiredTigerTestCase): conn_config = 'in_memory=false' write_10_values = [ ('10u', dict(write_10='u')), ('10h', dict(write_10='h')), ('10f', dict(write_10='f')), ('10m', dict(write_10='m')), ('10l', dict(write_10='l')), ] type_10_values = [ ('nil', dict(type_10=None)), ('upd', dict(type_10='upd')), ] write_20_values = [ ('20u', dict(write_20='u')), ('20h', dict(write_20='h')), ('20f', dict(write_20='f')), ('20m', dict(write_20='m')), ('20l', dict(write_20='l')), ] type_20_values = [ ('nil', dict(type_20=None)), ('upd', dict(type_20='upd')), ('del', dict(type_20='del')), ] write_30_values = [ ('30u', dict(write_30='u')), ('30h', dict(write_30='h')), ('30f', dict(write_30='f')), ('30m', dict(write_30='m')), ('30l', dict(write_30='l')), ] type_30_values = [ ('nil', dict(type_30=None)), ('upd', dict(type_30='upd')), ('del', dict(type_30='del')), ] evict_time_values = [ ('chk10', dict(evict_time=10)), ('chk20', dict(evict_time=20)), ('chk30', dict(evict_time=30)), ] rollback_time_values = [ ('roll15', dict(rollback_time=15)), ('roll25', dict(rollback_time=25)), ] def is_meaningful(name, vals): # The last write at evict time should be uniform, to get an RLE cell. if vals['evict_time'] == 10 and vals['write_10'] != 'u': return False if vals['evict_time'] == 20 and vals['write_20'] != 'u': return False if vals['evict_time'] == 30 and vals['write_30'] != 'u': return False # If the type is nil, the value must be uniform. if vals['type_10'] is None and vals['write_10'] != 'u': return False if vals['type_20'] is None and vals['write_20'] != 'u': return False if vals['type_30'] is None and vals['write_30'] != 'u': return False # Similarly, delete and heterogeneous doesn't make sense. if vals['type_10'] == 'del' and vals['write_10'] == 'h': return False if vals['type_20'] == 'del' and vals['write_20'] == 'h': return False if vals['type_20'] == 'del' and vals['write_30'] == 'h': return False # Both 10 and 20 shouldn't be nil. That's equivalent to 10 and 30 being nil. if vals['type_10'] is None and vals['type_20'] is None: return False # Avoid cases that delete nonexistent values. def deletes_nonexistent(): present = {} for k in range(2, 2 + my_rle_size): present[k] = False def adjust(ty, write): if ty is None: return for k in keys_of_write(write): if ty == 'upd': present[k] = True elif ty == 'del': if present[k]: present[k] = False else: raise KeyError adjust(vals['type_10'], vals['write_10']) adjust(vals['type_20'], vals['write_20']) adjust(vals['type_30'], vals['write_30']) try: deletes_nonexistent() except KeyError: return False return True scenarios = filter_scenarios( make_scenarios(write_10_values, type_10_values, write_20_values, type_20_values, write_30_values, type_30_values, evict_time_values, rollback_time_values), is_meaningful) value_z = "zzzzz" * 10 def writes(self, uri, s, expected, ty, write, value, ts): if ty is None: # do nothing at all return cursor = s.open_cursor(uri) s.begin_transaction() for k in keys_of_write(write): if ty == 'upd': myval = value + str(k) if write == 'h' else value cursor[k] = myval expected[k] = myval else: cursor.set_key(k) cursor.remove() del expected[k] s.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def evict(self, uri, s): # Evict the page to force reconciliation. evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)") s.begin_transaction() # Search the key to evict it. Use both bookends. v = evict_cursor[1] self.assertEqual(v, self.value_z) v = evict_cursor[2 + my_rle_size] self.assertEqual(v, self.value_z) self.assertEqual(evict_cursor.reset(), 0) s.rollback_transaction() evict_cursor.close() def check(self, uri, s, ts, expected): cursor = s.open_cursor(uri) s.begin_transaction('read_timestamp=' + self.timestamp_str(ts)) # endpoints should still be in place self.assertEqual(cursor[1], self.value_z) self.assertEqual(cursor[2 + my_rle_size], self.value_z) for k in range(2, 2 + my_rle_size): if k in expected: self.assertEqual(cursor[k], expected[k]) else: cursor.set_key(k) r = cursor.search() self.assertEqual(r, wiredtiger.WT_NOTFOUND) s.rollback_transaction() cursor.close() def test_rollback_to_stable25(self): # Create a table without logging. uri = "table:rollback_to_stable25" self.session.create(uri, 'key_format=r,value_format=S') # Pin oldest timestamp to 5. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5)) # Start stable timestamp at 5. self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5)) value_a = "aaaaa" * 10 value_b = "bbbbb" * 10 value_c = "ccccc" * 10 s = self.conn.open_session() # Write the endpoints at time 5. cursor = s.open_cursor(uri) s.begin_transaction() cursor[1] = self.value_z cursor[2 + my_rle_size] = self.value_z s.commit_transaction('commit_timestamp=' + self.timestamp_str(5)) self.evict(uri, s) cursor.close() # Do writes at time 10. expected = {} self.writes(uri, s, expected, self.type_10, self.write_10, value_a, 10) expected10 = expected.copy() # Evict at time 10 if requested. if self.evict_time == 10: self.evict(uri, s) # Do more writes at time 20. self.writes(uri, s, expected, self.type_20, self.write_20, value_b, 20) expected20 = expected.copy() # Evict at time 20 if requested. if self.evict_time == 20: self.evict(uri, s) # Do still more writes at time 30. self.writes(uri, s, expected, self.type_30, self.write_30, value_c, 30) expected30 = expected.copy() # Evict at time 30 if requested. if self.evict_time == 30: self.evict(uri, s) # Now roll back. self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.rollback_time)) self.conn.rollback_to_stable() if self.rollback_time < 20: expected20 = expected10 expected30 = expected10 elif self.rollback_time < 30: expected30 = expected20 # Now make sure we see what we expect. self.check(uri, s, 10, expected10) self.check(uri, s, 20, expected20) self.check(uri, s, 30, expected30)