def test_log_ts(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet( self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Open the object, configuring write_timestamp usage. uri = 'table:ts' config = ',write_timestamp_usage=' config += 'always' if self.always else 'never' self.session.create(uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format) + config) c = self.session.open_cursor(uri) # Commit with a timestamp. self.session.begin_transaction() c[ds.key(1)] = ds.value(1) self.session.breakpoint() self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) # Commit without a timestamp. self.session.begin_transaction() c[ds.key(2)] = ds.value(2) self.session.commit_transaction()
def test_search_eot(self): # Populate the tree and reopen the connection, forcing it to disk # and moving the records to an on-page format. ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format, value_format=self.value_format) ds.populate() self.reopen_conn() # Open a cursor. cursor = self.session.open_cursor(self.uri, None) # Search for a record at the end of the table, which should succeed. cursor.set_key(ds.key(100)) self.assertEqual(cursor.search(), 0) self.assertEqual(cursor.get_key(), ds.key(100)) self.assertEqual(cursor.get_value(), ds.value(100)) # Search-near for a record at the end of the table, which should # succeed, returning the last record. cursor.set_key(ds.key(100)) self.assertEqual(cursor.search_near(), 0) self.assertEqual(cursor.get_key(), ds.key(100)) self.assertEqual(cursor.get_value(), ds.value(100)) # Search for a record past the end of the table, which should fail. cursor.set_key(ds.key(200)) self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) # Search-near for a record past the end of the table, which should # succeed, returning the last record. cursor.set_key(ds.key(200)) self.assertEqual(cursor.search_near(), -1) self.assertEqual(cursor.get_key(), ds.key(100)) self.assertEqual(cursor.get_value(), ds.value(100))
def test_missing(self): ds = SimpleDataSet(self, self.uri, self.nentries, config=self.config, key_format=self.keyfmt) ds.populate() c = self.session.open_cursor(self.uri, None) for i in range(self.nentries + 3000, self.nentries + 5001): c[ds.key(i)] = ds.value(i) self.reopen_conn() c = self.session.open_cursor(self.uri, None) self.forward(c, ds, self.nentries + 5000, list(range(self.nentries + 1, self.nentries + 3000))) self.backward(c, ds, self.nentries + 5000, list(range(self.nentries + 1, self.nentries + 3000))) # Insert into the empty space so we test searching inserted items. for i in range(self.nentries + 1000, self.nentries + 2001): c[ds.key(i)] = ds.value(i) self.forward(c, ds, self.nentries + 5000, list(list(range(self.nentries + 1, self.nentries + 1000)) +\ list(range(self.nentries + 2001, self.nentries + 3000)))) self.backward(c, ds, self.nentries + 5000, list(list(range(self.nentries + 1, self.nentries + 1000)) +\ list(range(self.nentries + 2001, self.nentries + 3000))))
def test_timestamp_ts_then_nots(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet( self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Create the table with the key consistency checking turned on. That checking will verify # any individual key is always or never used with a timestamp. And if it is used with a # timestamp that the timestamps are in increasing order for that key. uri = 'table:ts' self.session.create(uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format) + ',write_timestamp_usage=ordered') c = self.session.open_cursor(uri) key = ds.key(5) self.session.begin_transaction() c[key] = ds.value(11) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) self.session.begin_transaction() c[key] = ds.value(12) msg ='/configured to always use timestamps once they are first used/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg)
def test_search_invisible_two(self): # Populate the tree and reopen the connection, forcing it to disk # and moving the records to an on-page format. ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format, value_format=self.value_format) ds.populate() self.reopen_conn() # Add some additional visible records. cursor = self.session.open_cursor(self.uri, None) for i in range(100, 120): cursor[ds.key(i)] = ds.value(i) cursor.close() # Begin a transaction, and add some additional records. self.session.begin_transaction() cursor = self.session.open_cursor(self.uri, None) for i in range(120, 140): cursor[ds.key(i)] = ds.value(i) # Open a separate session and cursor. s = self.conn.open_session() cursor = s.open_cursor(self.uri, None) # Search for an invisible record. cursor.set_key(ds.key(130)) if self.empty: # Invisible updates to fixed-length column-store objects are # invisible to the reader, but the fact that they exist past # the end of the initial records causes the instantiation of # empty records: confirm successful return of an empty row. cursor.search() self.assertEqual(cursor.get_key(), 130) self.assertEqual(cursor.get_value(), 0) else: # Otherwise, we should not find any matching records. self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) # Search-near for an invisible record, which should succeed, returning # the last visible record. cursor.set_key(ds.key(130)) cursor.search_near() if self.empty: # Invisible updates to fixed-length column-store objects are # invisible to the reader, but the fact that they exist past # the end of the initial records causes the instantiation of # empty records: confirm successful return of an empty row. cursor.search() self.assertEqual(cursor.get_key(), 130) self.assertEqual(cursor.get_value(), 0) else: # Otherwise, we should find the closest record for which we can see # the value. self.assertEqual(cursor.get_key(), ds.key(119)) self.assertEqual(cursor.get_value(), ds.value(119))
def test_timestamp_ts_order(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet(self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Create the table with the key consistency checking turned on. That checking will verify # any individual key is always or never used with a timestamp. And if it is used with a # timestamp that the timestamps are in increasing order for that key. uri = 'table:ts' self.session.create( uri, 'key_format={},value_format={}'.format( self.key_format, self.value_format) + ',write_timestamp_usage=ordered') c = self.session.open_cursor(uri) key1 = ds.key(6) key2 = ds.key(7) self.session.begin_transaction() self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(30)) c[key1] = ds.value(14) c[key2] = ds.value(15) self.session.commit_transaction() self.assertEquals(c[key1], ds.value(14)) self.assertEquals(c[key2], ds.value(15)) self.session.begin_transaction() c[key1] = ds.value(16) self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(31)) c[key2] = ds.value(17) self.session.commit_transaction() self.assertEquals(c[key1], ds.value(16)) self.assertEquals(c[key2], ds.value(17)) self.session.begin_transaction() c[key1] = ds.value(18) c[key2] = ds.value(19) self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(32)) self.session.commit_transaction() self.assertEquals(c[key1], ds.value(18)) self.assertEquals(c[key2], ds.value(19)) self.session.begin_transaction() c[key1] = ds.value(20) c[key2] = ds.value(21) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(33)) self.assertEquals(c[key1], ds.value(20)) self.assertEquals(c[key2], ds.value(21))
def test_in_memory_ts(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet(self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Open the object, configuring write_timestamp usage. uri = 'table:ts' config = ',' + self.obj_config config += ',write_timestamp_usage=' config += 'ordered' if self.always else 'never' self.session.breakpoint() self.session.create( uri, 'key_format={},value_format={}'.format( self.key_format, self.value_format) + config) c = self.session.open_cursor(uri) # Commit with a timestamp. self.session.begin_transaction() c[ds.key(1)] = ds.value(1) if self.always == True or self.obj_ignore == True: self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) else: msg = '/unexpected timestamp usage/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: self.session.commit_transaction( 'commit_timestamp=' + self.timestamp_str(1)), msg) # Commit without a timestamp (but first with a timestamp if in ordered mode so we get # a failure). if self.always: self.session.begin_transaction() c[ds.key(2)] = ds.value(2) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() c[ds.key(2)] = ds.value(2) if self.always == False or self.obj_ignore == True: self.session.commit_transaction() else: msg = '/no timestamp provided/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg)
def test_modify_abort(self): ds = SimpleDataSet(self, self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() # Start a transaction. self.session.begin_transaction("isolation=snapshot") # Insert a new record. c = self.session.open_cursor(self.uri, None) c.set_key(ds.key(30)) c.set_value(ds.value(30)) self.assertEquals(c.insert(), 0) # Test that we can successfully modify our own record. mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(30)) mods = self.fix_mods(mods) self.assertEqual(c.modify(mods), 0) # Test that another transaction cannot modify our uncommitted record. xs = self.conn.open_session() xc = xs.open_cursor(self.uri, None) xs.begin_transaction("isolation=snapshot") xc.set_key(ds.key(30)) xc.set_value(ds.value(30)) mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) mods = self.fix_mods(mods) xc.set_key(ds.key(30)) self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND) xs.rollback_transaction() # Rollback our transaction. self.session.rollback_transaction() # Test that we can't modify our aborted insert. self.session.begin_transaction("isolation=snapshot") mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) mods = self.fix_mods(mods) c.set_key(ds.key(30)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction()
def test_smoke(self): ds = SimpleDataSet(self, self.uri, self.nentries, config=self.config, key_format=self.keyfmt) ds.populate() self.reopen_conn() c = self.session.open_cursor(self.uri, None) c.set_key(ds.key(100)) self.assertEqual(c.search(), 0) self.assertEqual(c.get_value(), ds.value(100)) c.set_key(ds.key(101)) self.assertEqual(c.search(), 0) self.assertEqual(c.get_value(), ds.value(101)) c.set_key(ds.key(9999)) self.assertEqual(c.search(), 0) self.assertEqual(c.get_value(), ds.value(9999))
def test_prepare18(self): uri = "table:prepare18" ds = SimpleDataSet(self, uri, 100, key_format='S', value_format='S') ds.populate() cursor = self.session.open_cursor(uri, None) self.session.begin_transaction() cursor[ds.key(10)] = ds.value(20) self.session.commit_transaction() self.session.begin_transaction() cursor[ds.key(10)] = ds.value(20) msg = '/a prepared transaction cannot include a logged table/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: self.session.prepare_transaction('prepare_timestamp=1'), msg)
def test_wtu_never(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet( self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Open the object, configuring write_timestamp usage. uri = 'table:ts' self.session.create(uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format) + ',write_timestamp_usage=never') c = self.session.open_cursor(uri) self.session.begin_transaction() c[ds.key(7)] = ds.value(8) # Commit with a timestamp. if self.with_ts: # Check both an explicit timestamp set and a set at commit. commit_ts = 'commit_timestamp=' + self.timestamp_str(10) if not self.commit_ts: self.session.timestamp_transaction(commit_ts) commit_ts = '' msg = '/set when disallowed/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(commit_ts), msg) # Commit without a timestamp. else: self.session.commit_transaction()
def test_insert_over_delete_replace(self): msg = '/WT_CACHE_FULL.*/' ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, ds.populate, msg) cursor = self.session.open_cursor(self.uri, None) cursor.prev() last_key = int(cursor.get_key()) # Now that the database contains as much data as will fit into # the configured cache, verify removes succeed. cursor = self.session.open_cursor(self.uri, None) for i in range(1, last_key / 4, 1): cursor.set_key(ds.key(i)) cursor.remove() cursor.reset() # Spin inserting to give eviction a chance to reclaim space inserted = False for i in range(1, 1000): try: cursor[ds.key(1)] = ds.value(1) except wiredtiger.WiredTigerError: cursor.reset() sleep(1) continue inserted = True break self.assertTrue(inserted)
def test_insert_over_delete_replace(self): msg = '/WT_CACHE_FULL.*/' ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, ds.populate, msg) cursor = self.session.open_cursor(self.uri, None) cursor.prev() last_key = int(cursor.get_key()) # Now that the database contains as much data as will fit into # the configured cache, verify removes succeed. cursor = self.session.open_cursor(self.uri, None) for i in range(1, last_key // 4, 1): cursor.set_key(ds.key(i)) cursor.remove() cursor.reset() # Spin inserting to give eviction a chance to reclaim space sleeps = 0 inserted = False for i in range(1, 1000): try: cursor[ds.key(1)] = ds.value(1) except wiredtiger.WiredTigerError: cursor.reset() sleeps = sleeps + 1 self.assertLess(sleeps, 60 * 5) sleep(1) continue inserted = True break self.assertTrue(inserted)
def test_modify_abort(self): ds = SimpleDataSet(self, self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt) ds.populate() # Start a transaction. self.session.begin_transaction() # Insert a new record. c = self.session.open_cursor(self.uri, None) c.set_key(ds.key(30)) c.set_value(ds.value(30)) self.assertEquals(c.insert(), 0) # Test that we can successfully modify our own record. mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(30)) self.assertEqual(c.modify(mods), 0) # Test that another transaction cannot modify our uncommitted record. xs = self.conn.open_session() xc = xs.open_cursor(self.uri, None) xs.begin_transaction() xc.set_key(ds.key(30)) xc.set_value(ds.value(30)) mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) xc.set_key(ds.key(30)) self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND) xs.rollback_transaction() # Rollback our transaction. self.session.rollback_transaction() # Test that we can't modify our aborted insert. self.session.begin_transaction() mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(30)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction()
def test_alter(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet(self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Open the object, configuring "never" timestamp usage. # Check it. # Switch the object to "ordered" usage. # Check it. uri = 'table:ts' self.session.create( uri, 'key_format={},value_format={}'.format( self.key_format, self.value_format) + ',write_timestamp_usage=never') c = self.session.open_cursor(uri) self.session.begin_transaction() c[ds.key(10)] = ds.value(10) msg = '/set when disallowed by table configuration/' self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: self.session.commit_transaction('commit_timestamp=' + self. timestamp_str(10)), msg) c.close() self.session.alter(uri, 'write_timestamp_usage=ordered') c = self.session.open_cursor(uri) self.session.begin_transaction() c[ds.key(10)] = ds.value(10) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(11)) self.session.begin_transaction() c[ds.key(10)] = ds.value(10) msg = '/always use timestamps/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg)
def test_dup_key(self): uri = 'table:dup_key' ds = SimpleDataSet(self, uri, 100, key_format=self.key_format, value_format=self.value_format) ds.populate() if self.reopen: self.reopen_conn() c = self.session.open_cursor(uri, None, 'overwrite=false') c.set_key(ds.key(10)) c.set_value(ds.value(20)) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, lambda: c.insert(), '/WT_DUPLICATE_KEY/') self.assertEqual(c.get_value(), ds.value(10))
def test_in_memory_ts(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet(self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Open the object, configuring write_timestamp usage. uri = 'table:ts' config = ',' + self.obj_config config += ',write_timestamp_usage=' config += 'always' if self.always else 'never' config += ',assert=(write_timestamp=on)' self.session.breakpoint() self.session.create( uri, 'key_format={},value_format={}'.format( self.key_format, self.value_format) + config) c = self.session.open_cursor(uri) # Commit with a timestamp. self.session.begin_transaction() c[ds.key(1)] = ds.value(1) if self.always == True or self.obj_ignore == True: self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) else: with self.expectedStderrPattern('unexpected timestamp usage'): self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(1)) # Commit without a timestamp. self.session.begin_transaction() c[ds.key(2)] = ds.value(2) if self.always == False or self.obj_ignore == True: self.session.commit_transaction() else: with self.expectedStderrPattern('unexpected timestamp usage'): self.session.commit_transaction()
def test_read_timestamp(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet(self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Open the object, configuring read timestamp usage. uri = 'table:ts' self.session.create( uri, 'key_format={},value_format={}'.format( self.key_format, self.value_format) + ',assert=(read_timestamp=' + self.read_ts + ')') c = self.session.open_cursor(uri) key = ds.key(10) value = ds.value(10) # Insert a data item at a timestamp (although it doesn't really matter). self.session.begin_transaction() self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(10)) c[key] = value self.session.timestamp_transaction() self.session.commit_transaction() # Try reading without a timestamp. self.session.begin_transaction() c.set_key(key) if self.read_ts != 'always': self.assertEquals(c.search(), 0) self.assertEqual(c.get_value(), value) else: msg = '/read timestamps required and none set/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c.search(), msg) self.session.rollback_transaction() # Try reading with a timestamp. self.session.begin_transaction() self.session.timestamp_transaction('read_timestamp=20') c.set_key(key) if self.read_ts != 'never': self.assertEquals(c.search(), 0) self.assertEqual(c.get_value(), value) else: msg = '/read timestamps disallowed/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c.search(), msg) self.session.rollback_transaction()
def test_rollback_to_stable33(self): uri = "table:rollback_to_stable33" ds_config = ',log=(enabled=true)' if self.logged else ',log=(enabled=false)' ds = SimpleDataSet(self, uri, 500, key_format=self.key_format, value_format=self.value_format, config=ds_config) ds.populate() # Make changes at timestamp 30. c = self.session.open_cursor(uri, None, None) self.session.begin_transaction() c[ds.key(10)] = ds.value(100) c[ds.key(11)] = ds.value(101) c[ds.key(12)] = ds.value(102) self.session.commit_transaction('commit_timestamp=30') c.close() # Set stable to 20 and rollback. self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(20)) self.conn.rollback_to_stable() # Objects with logging enabled should not be rolled back, objects without logging enabled # should have their updates rolled back. c = self.session.open_cursor(uri, None) if self.logged: self.assertEquals(c[ds.key(10)], ds.value(100)) self.assertEquals(c[ds.key(11)], ds.value(101)) self.assertEquals(c[ds.key(12)], ds.value(102)) else: self.assertEquals(c[ds.key(10)], ds.value(10)) self.assertEquals(c[ds.key(11)], ds.value(11)) self.assertEquals(c[ds.key(12)], ds.value(12))
def test_column_store_gap_traverse(self): uri = 'table:gap' # Initially just create tables. ds = SimpleDataSet(self, uri, 0, key_format='r') ds.populate() cursor = self.session.open_cursor(uri, None, None) self.nentries = 0 # Create a column store with key gaps. The particular values aren't # important, we just want some gaps. v = [ 1000, 1001, 2000, 2001] for i in v: cursor[ds.key(i)] = ds.value(i) self.nentries += 1 # In-memory cursor forward, backward. self.forward(cursor, v) self.backward(cursor, list(reversed(v))) self.reopen_conn() cursor = self.session.open_cursor(uri, None, None) # Disk page cursor forward, backward. self.forward(cursor, v) self.backward(cursor, list(reversed(v))) # Insert some new records, so there are in-memory updates and an # on disk image. Put them in the middle of the existing values # so the traversal walks to them. v2 = [ 1500, 1501 ] for i in v2: cursor[ds.key(i)] = ds.value(i) self.nentries += 1 # Tell the validation what to expect. v = [ 1000, 1001, 1500, 1501, 2000, 2001 ] self.forward(cursor, v) self.backward(cursor, list(reversed(v)))
def test_flcs(self): uri = "table:test_flcs05" nrows = 44 ds = SimpleDataSet( self, uri, nrows, key_format='r', value_format='6t', config='leaf_page_max=4096') ds.populate() updatekey1 = 33 updatekey2 = 37 appendkey1 = nrows + 10 cursor = self.session.open_cursor(uri, None, 'overwrite=false') # Write a few records. #self.session.begin_transaction() #for i in range(1, nrows + 1): # self.prout("foo {}".format(i)) # cursor.set_key(i) # cursor.set_value(i) # self.assertEqual(cursor.update(), 0) #self.session.commit_transaction() # There are five cases: # 1. A nonzero value. # 2. A zero/deleted value that's been reconciled. # 3. A zero/deleted value that hasn't been reconciled but that's on/over a page. # 4. A zero/deleted value that hasn't been reconciled and is in the append list. # 5. A value that only exists implicitly becaues the append list has gone past it. # Nonzero value. self.tryread(cursor, updatekey1, ds.value(updatekey1)) # Deleted value that hasn't been reconciled. cursor.set_key(updatekey2) self.assertEqual(cursor.remove(), 0) self.tryread(cursor, updatekey2, 0) # Deleted value that has been reconciled. self.evict(ds.uri, updatekey2, 0) self.tryread(cursor, updatekey2, 0) # Deleted value in the append list. cursor[appendkey1] = appendkey1 cursor.set_key(appendkey1) self.assertEqual(cursor.remove(), 0) self.tryread(cursor, appendkey1, 0) # Implicit value. self.tryread(cursor, appendkey1 - 1, 0)
def test_timestamp_inconsistent_update(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet( self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Create the table with the key consistency checking turned on. That checking will verify # any individual key is always or never used with a timestamp. And if it is used with a # timestamp that the timestamps are in increasing order for that key. uri = 'table:ts' self.session.create(uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format) + ',write_timestamp_usage=ordered') c = self.session.open_cursor(uri) key = ds.key(1) # Insert an item at timestamp 2. self.session.begin_transaction() c[key] = ds.value(1) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) # Upate the data item at timestamp 1, which should fail. self.session.begin_transaction() self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) c[key] = ds.value(2) msg = '/updates a value with an older timestamp/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg) # Make sure we can successfully add a different key at timestamp 1. self.session.begin_transaction() self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) c[ds.key(2)] = ds.value(3) self.session.commit_transaction() # Insert key1 at timestamp 10 and key2 at 15. Then update both keys in one transaction at # timestamp 13, and we should get a complaint about usage. key1 = ds.key(3) key2 = ds.key(4) self.session.begin_transaction() c[key1] = ds.value(3) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(10)) self.session.begin_transaction() c[key2] = ds.value(4) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) self.session.begin_transaction() self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(13)) c[key1] = ds.value(5) c[key2] = ds.value(6) msg = '/updates a value with an older timestamp/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg)
def test_always_never(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet(self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Open the object, configuring write_timestamp usage. uri = 'table:ts' self.session.create( uri, 'key_format={},value_format={}'.format( self.key_format, self.value_format) + ',write_timestamp_usage=' + self.write_timestamp + ',' + ',assert=(write_timestamp=' + self.assert_ts + ')') c = self.session.open_cursor(uri) self.session.begin_transaction() c[ds.key(7)] = ds.value(8) # Commit with a timestamp. if self.with_ts: # Check both an explicit timestamp set and a set at commit. commit_ts = 'commit_timestamp=' + self.timestamp_str(10) if not self.commit_ts: self.session.timestamp_transaction(commit_ts) commit_ts = '' if self.assert_ts == 'off' or self.write_timestamp == 'always': self.session.commit_transaction(commit_ts) else: with self.expectedStderrPattern('set when disallowed'): self.session.commit_transaction(commit_ts) # Commit without a timestamp. else: if self.assert_ts == 'off' or self.write_timestamp == 'never': self.session.commit_transaction() else: with self.expectedStderrPattern('timestamp required by table'): self.session.commit_transaction()
def test_column_store_gap(self): uri = 'table:gap' # Initially just create tables. ds = SimpleDataSet(self, uri, 0, key_format='r') ds.populate() cursor = self.session.open_cursor(uri, None, None) self.nentries = 0 # Create a column-store table with large gaps in the name-space. v = [ 1000, 2000000000000, 30000000000000 ] for i in v: cursor[ds.key(i)] = ds.value(i) self.nentries += 1 # In-memory cursor forward, backward. self.forward(cursor, v) self.backward(cursor, list(reversed(v))) self.reopen_conn() cursor = self.session.open_cursor(uri, None, None) # Disk page cursor forward, backward. self.forward(cursor, v) self.backward(cursor, list(reversed(v)))
def test_search_invisible_one(self): # Populate the tree. ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format, value_format=self.value_format) ds.populate() # Delete a range of records. for i in range(5, 10): cursor = self.session.open_cursor(self.uri, None) cursor.set_key(ds.key(i)) self.assertEqual(cursor.remove(), 0) # Reopen the connection, forcing it to disk and moving the records to # an on-page format. self.reopen_conn() # Add updates to the existing records (in both the deleted an undeleted # range), as well as some new records after the end. Put the updates in # a separate transaction so they're invisible to another cursor. self.session.begin_transaction() cursor = self.session.open_cursor(self.uri, None) for i in range(5, 10): cursor[ds.key(i)] = ds.value(i + 1000) for i in range(30, 40): cursor[ds.key(i)] = ds.value(i + 1000) for i in range(100, 140): cursor[ds.key(i)] = ds.value(i + 1000) # Open a separate session and cursor. s = self.conn.open_session() cursor = s.open_cursor(self.uri, None) # Search for an existing record in the deleted range, should not find # it. for i in range(5, 10): cursor.set_key(ds.key(i)) if self.empty: # Fixed-length column-store rows always exist. self.assertEqual(cursor.search(), 0) self.assertEqual(cursor.get_key(), i) self.assertEqual(cursor.get_value(), 0) else: self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) # Search for an existing record in the updated range, should see the # original value. for i in range(30, 40): cursor.set_key(ds.key(i)) self.assertEqual(cursor.search(), 0) self.assertEqual(cursor.get_key(), ds.key(i)) # Search for a added record, should not find it. for i in range(120, 130): cursor.set_key(ds.key(i)) if self.empty: # Invisible updates to fixed-length column-store objects are # invisible to the reader, but the fact that they exist past # the end of the initial records causes the instantiation of # empty records: confirm successful return of an empty row. self.assertEqual(cursor.search(), 0) self.assertEqual(cursor.get_key(), i) self.assertEqual(cursor.get_value(), 0) else: # Otherwise, we should not find any matching records. self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) # Search-near for an existing record in the deleted range, should find # the next largest record. (This depends on the implementation behavior # which currently includes a bias to prefix search.) for i in range(5, 10): cursor.set_key(ds.key(i)) if self.empty: # Fixed-length column-store rows always exist. self.assertEqual(cursor.search_near(), 0) self.assertEqual(cursor.get_key(), i) self.assertEqual(cursor.get_value(), 0) else: self.assertEqual(cursor.search_near(), 1) self.assertEqual(cursor.get_key(), ds.key(10)) # Search-near for an existing record in the updated range, should see # the original value. for i in range(30, 40): cursor.set_key(ds.key(i)) self.assertEqual(cursor.search_near(), 0) self.assertEqual(cursor.get_key(), ds.key(i)) # Search-near for an added record, should find the previous largest # record. for i in range(120, 130): cursor.set_key(ds.key(i)) if self.empty: # Invisible updates to fixed-length column-store objects are # invisible to the reader, but the fact that they exist past # the end of the initial records causes the instantiation of # empty records: confirm successful return of an empty row. self.assertEqual(cursor.search_near(), 0) self.assertEqual(cursor.get_key(), i) self.assertEqual(cursor.get_value(), 0) else: self.assertEqual(cursor.search_near(), -1) self.assertEqual(cursor.get_key(), ds.key(100))
def test_truncate_fast_delete(self): uri = self.type + self.name ''' print '===== run:' print 'config:', self.config + self.keyfmt, \ 'overflow=', self.overflow, \ 'readafter=', self.readafter, 'readbefore=', self.readbefore, \ 'writeafter=', self.writeafter, 'writebefore=', self.writebefore, \ 'commit=', self.commit ''' # Create the object. ds = SimpleDataSet(self, uri, self.nentries, config=self.config, key_format=self.keyfmt) ds.populate() # Optionally add a few overflow records so we block fast delete on # those pages. if self.overflow: cursor = self.session.open_cursor(uri, None, 'overwrite=false') for i in range(1, self.nentries, 3123): cursor.set_key(ds.key(i)) cursor.set_value(ds.value(i)) cursor.update() cursor.close() # Close and re-open it so we get a disk image, not an insert skiplist. self.reopen_conn() # Optionally read/write a few rows before truncation. if self.readbefore or self.writebefore: cursor = self.session.open_cursor(uri, None, 'overwrite=false') if self.readbefore: for i in range(1, self.nentries, 737): cursor.set_key(ds.key(i)) cursor.search() if self.writebefore: for i in range(1, self.nentries, 988): cursor.set_key(ds.key(i)) cursor.set_value(ds.value(i + 100)) cursor.update() cursor.close() # Begin a transaction, and truncate a big range of rows. self.session.begin_transaction(None) start = self.session.open_cursor(uri, None) start.set_key(ds.key(10)) end = self.session.open_cursor(uri, None) end.set_key(ds.key(self.nentries - 10)) self.session.truncate(None, start, end, None) start.close() end.close() # Optionally read/write a few rows after truncation. if self.readafter or self.writeafter: cursor = self.session.open_cursor(uri, None, 'overwrite=false') if self.readafter: for i in range(1, self.nentries, 1123): cursor.set_key(ds.key(i)) cursor.search() if self.writeafter: for i in range(1, self.nentries, 621): cursor.set_key(ds.key(i)) cursor.set_value(ds.value(i + 100)) cursor.update() cursor.close() # A cursor involved in the transaction should see the deleted records. # The number 19 comes from deleting row 10 (inclusive), to row N - 10, # exclusive, or 9 + 10 == 19. remaining = 19 cursor = self.session.open_cursor(uri, None) self.cursor_count(cursor, remaining) cursor.close() # A separate, read_committed cursor should not see the deleted records. self.outside_count("isolation=read-committed", self.nentries) # A separate, read_uncommitted cursor should see the deleted records. self.outside_count("isolation=read-uncommitted", remaining) # Commit/rollback the transaction. if self.commit: self.session.commit_transaction() else: self.session.rollback_transaction() # Check a read_committed cursor sees the right records. cursor = self.session.open_cursor(uri, None) if self.commit: self.cursor_count(cursor, remaining) else: self.cursor_count(cursor, self.nentries) cursor.close()
def test_timestamp_alter(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet(self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) cfg_on = 'write_timestamp_usage=ordered' cfg_off = 'write_timestamp_usage=none' # Create the table without the key consistency checking turned on. # Create a few items breaking the rules. # Then alter the setting and verify the inconsistent usage is detected. uri = 'file:assert06' self.session.create( uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format)) c = self.session.open_cursor(uri) # Insert a data item at timestamp 2. key = ds.key(1) self.session.begin_transaction() c[key] = ds.value(1) self.apply_timestamps(2, True) self.session.commit_transaction() # Modify the data item at timestamp 1, illegally moving the timestamp backward. self.session.begin_transaction() c[key] = ds.value(2) self.apply_timestamps(1, True) self.session.commit_transaction() # Insert a non-timestamped item. # Then illegally modify with a timestamp. # Then illegally modify without a timestamp. key = ds.key(2) self.session.begin_transaction() c[key] = ds.value(3) self.session.commit_transaction() self.session.begin_transaction() c[key] = ds.value(4) self.apply_timestamps(2, True) self.session.commit_transaction() self.session.begin_transaction() c[key] = ds.value(5) self.session.commit_transaction() # Now alter the setting and make sure we detect incorrect usage. # We must move the oldest timestamp forward in order to alter, otherwise alter closing the # file will fail with EBUSY. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(2)) c.close() self.session.alter(uri, cfg_on) c = self.session.open_cursor(uri) # Update at timestamp 5, then detect not using a timestamp. key = ds.key(3) self.session.begin_transaction() c[key] = ds.value(6) self.apply_timestamps(5, True) self.session.commit_transaction() self.session.begin_transaction() c[key] = ds.value(6) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), self.msg_usage) # Detect using a timestamp on a non-timestamp key. We must first use a non-timestamped # operation on the key in order to violate the key consistency condition in the following # transaction. key = ds.key(4) self.session.begin_transaction() c[key] = ds.value(7) self.session.commit_transaction() self.session.begin_transaction() c[key] = ds.value(8) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3)) # Test to make sure that key consistency can be turned off after turning it on. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(4)) c.close() self.session.alter(uri, cfg_off) c = self.session.open_cursor(uri) # Detection is off we can successfully change the same key with and without a timestamp. key = ds.key(5) self.session.begin_transaction() c[key] = ds.value(9) self.session.commit_transaction() self.session.begin_transaction() c[key] = ds.value(1) self.apply_timestamps(6, True) self.session.commit_transaction()
def test_timestamp_usage(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet(self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Create the table with the key consistency checking turned on. That checking will verify # any individual key is always or never used with a timestamp. And if it is used with a # timestamp that the timestamps are in increasing order for that key. uri = 'file:assert06' self.session.create( uri, 'key_format={},value_format={},'.format( self.key_format, self.value_format) + 'write_timestamp_usage=ordered,assert=(write_timestamp=on)') c = self.session.open_cursor(uri) # Insert a data item at timestamp 2. self.session.begin_transaction() c[ds.key(1)] = ds.value(1) self.apply_timestamps(2, True) self.session.commit_transaction() # Make sure we can successfully add a different key at timestamp 1. self.session.begin_transaction() c[ds.key(2)] = ds.value(2) self.apply_timestamps(1, True) self.session.commit_transaction() # Insert key_ts3 at timestamp 10 and key_ts4 at 15, then modify both keys in one transaction # at timestamp 13, which should result in an error message. c = self.session.open_cursor(uri) self.session.begin_transaction() c[ds.key(3)] = ds.value(3) self.apply_timestamps(10, True) self.session.commit_transaction() self.session.begin_transaction() c[ds.key(4)] = ds.value(4) self.apply_timestamps(15, True) self.session.commit_transaction() self.session.begin_transaction() c[ds.key(3)] = ds.value(5) c[ds.key(4)] = ds.value(6) self.apply_timestamps(13, False) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), '/unexpected timestamp usage/') self.assertEquals(c[ds.key(3)], ds.value(3)) self.assertEquals(c[ds.key(4)], ds.value(4)) # Modify a key previously used with timestamps without one. We should get the inconsistent # usage message. key = ds.key(5) self.session.begin_transaction() c[key] = ds.value(7) self.apply_timestamps(14, True) self.session.commit_transaction() self.session.begin_transaction() c[key] = ds.value(8) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), self.msg_usage) # Set the timestamp in the beginning, middle or end of the transaction. key = ds.key(6) self.session.begin_transaction() self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(16)) c[key] = ds.value(9) self.session.commit_transaction() self.assertEquals(c[key], ds.value(9)) key = ds.key(7) self.session.begin_transaction() c[key] = ds.value(10) c[key] = ds.value(11) self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(17)) c[key] = ds.value(12) c[key] = ds.value(13) self.session.commit_transaction() self.assertEquals(c[key], ds.value(13)) key = ds.key(8) self.session.begin_transaction() c[key] = ds.value(14) self.apply_timestamps(18, True) self.session.commit_transaction() self.assertEquals(c[key], ds.value(14)) # Confirm it is okay to set the durable timestamp on the commit call. key = ds.key(9) self.session.begin_transaction() c[key] = ds.value(15) c[key] = ds.value(16) self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(22)) self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(22)) self.session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(22)) self.session.commit_transaction() # Confirm that rolling back after preparing doesn't fire an assertion. key = ds.key(10) self.session.begin_transaction() c[key] = ds.value(17) self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(30)) self.session.rollback_transaction()
def test_alter_inconsistent_update(self): if wiredtiger.diagnostic_build(): self.skipTest('requires a non-diagnostic build') # Create an object that's never written, it's just used to generate valid k/v pairs. ds = SimpleDataSet( self, 'file:notused', 10, key_format=self.key_format, value_format=self.value_format) # Create the table without the key consistency checking turned on. # Create a few items breaking the rules. Then alter the setting and # verify the inconsistent usage is detected. uri = 'table:ts' self.session.create(uri, 'key_format={},value_format={}'.format(self.key_format, self.value_format)) c = self.session.open_cursor(uri) key = ds.key(10) # Insert a data item at timestamp 2. self.session.begin_transaction() self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(2)) c[key] = ds.value(10) self.session.commit_transaction() # Update the data item at timestamp 1. self.session.begin_transaction() self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(1)) c[key] = ds.value(11) self.session.commit_transaction() key = ds.key(12) # Insert a non-timestamped item, then update with a timestamp and then without a timestamp. self.session.begin_transaction() c[key] = ds.value(12) self.session.commit_transaction() self.session.begin_transaction() c[key] = ds.value(13) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2)) self.session.begin_transaction() c[key] = ds.value(14) self.session.commit_transaction() # Now alter the setting and make sure we detect incorrect usage. We must move the oldest # timestamp forward in order to alter, otherwise alter will fail with EBUSY. c.close() self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) self.session.alter(uri, 'write_timestamp_usage=ordered') c = self.session.open_cursor(uri) key = ds.key(15) # Detect decreasing timestamp. self.session.begin_transaction() c[key] = ds.value(15) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(15)) msg = '/with an older timestamp/' self.session.begin_transaction() self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(14)) c[key] = ds.value(16) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg) # Detect not using a timestamp. msg = '/use timestamps once they are first used/' self.session.begin_transaction() c[key] = ds.value(17) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.commit_transaction(), msg)
def test_truncate_simple(self): uri = self.type + self.name # layout: # the number of initial skipped records # the number of initial inserted records # the number of trailing skipped records # the number of trailing inserted records layout = [ # simple set of rows (0, 0, 0, 0), # trailing append list, no delete point overlap (0, 0, 0, self.skip - 3), # trailing append list, delete point overlap (0, 0, 0, self.skip + 3), # trailing skipped list, no delete point overlap (0, 0, self.skip - 3, 1), # trailing skipped list, delete point overlap (0, 0, self.skip + 3, 1), # leading insert list, no delete point overlap (0, self.skip - 3, 0, 0), # leading insert list, delete point overlap (0, self.skip + 3, 0, 0), # leading skipped list, no delete point overlap (self.skip - 3, 1, 0, 0), # leading skipped list, delete point overlap (self.skip + 3, 1, 0, 0), ] # list: truncation patterns applied on top of the layout. # # begin and end: -1 means pass None for the cursor arg to truncate. An # integer N, with 1 <= N < self.nentries, truncates from/to a cursor # positioned at that row. list = [ (-1, self.nentries), # begin to end, begin = None (1, -1), # begin to end, end = None (1, self.nentries), # begin to end (-1, self.nentries - self.skip), # begin to middle, begin = None (1, self.nentries - self.skip), # begin to middle (self.skip, -1), # middle to end, end = None (self.skip, self.nentries), # middle to end ( self.skip, # middle to different middle self.nentries - self.skip), (1, 1), # begin to begin (self.nentries, self.nentries), # end to end (self.skip, self.skip) # middle to same middle ] # Using this data set to compare only, it doesn't create or populate. ds = SimpleDataSet(self, uri, 0, key_format=self.keyfmt, value_format=self.valuefmt, config=self.config) # Build the layout we're going to test total = self.nentries for begin_skipped, begin_insert, end_skipped, end_insert in layout: # skipped records require insert/append records if begin_skipped and not begin_insert or \ end_skipped and not end_insert: raise AssertionError('test error: skipped set without insert') for begin, end in list: ''' print '===== run:' print 'key:', self.keyfmt, 'begin:', begin, 'end:', end print 'total: ', total, \ 'begin_skipped:', begin_skipped, \ 'begin_insert:', begin_insert, \ 'end_skipped:', end_skipped, \ 'end_insert:', end_insert ''' # Build a dictionary of what the object should look like for # later comparison expected = {} # Create the object. self.session.create( uri, self.config + ',key_format=' + self.keyfmt + ',value_format=' + self.valuefmt) # Insert the records that aren't skipped or inserted. start = begin_skipped + begin_insert stop = self.nentries - (end_skipped + end_insert) cursor = self.session.open_cursor(uri, None) for i in range(start + 1, stop + 1): k = ds.key(i) v = ds.value(i) cursor[k] = v expected[k] = [v] cursor.close() # Optionally close and re-open the object to get a disk image # instead of a big insert list. if self.reopen: self.reopen_conn() # Optionally insert initial skipped records. cursor = self.session.open_cursor(uri, None, "overwrite") start = 0 for i in range(0, begin_skipped): start += 1 k = ds.key(start) expected[k] = [0] # Optionally insert leading records. for i in range(0, begin_insert): start += 1 k = ds.key(start) v = ds.value(start) cursor[k] = v expected[k] = [v] # Optionally insert trailing skipped records. for i in range(0, end_skipped): stop += 1 k = ds.key(stop) expected[k] = [0] # Optionally insert trailing records. for i in range(0, end_insert): stop += 1 k = ds.key(stop) v = ds.value(stop) cursor[k] = v expected[k] = [v] cursor.close() self.truncateRangeAndCheck(ds, uri, begin, end, expected) self.session.drop(uri, None)
def test_insert_over_delete_replace(self): msg = '/WT_CACHE_FULL.*/' ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, ds.populate, msg) cursor = self.session.open_cursor(self.uri, None) cursor.prev() last_key = int(cursor.get_key()) # This test fails on FLCS when the machine is under heavy load: it gets WT_CACHE_FULL # forever in the bottom loop and eventually fails there. This is at least partly because # in FLCS removing values does not recover space (deleted values are stored as 0). # # I think what happens is that under sufficient load the initial fill doesn't fail until # all the pages in it have already been reconciled. Then since removing some of the rows # in the second step doesn't free any space up, there's no space for more updates and # the bottom loop eventually fails. When not under load, at least one page in the # initial fill isn't reconciled until after the initial fill stops; it gets reconciled # afterwards and that frees up enough space to do the rest of the writes. (Because # update structures are much larger than FLCS values, which are one byte, reconciling a # page with pending updates recovers a lot of space.) # # There does not seem to currently be any way to keep this from happening. (If we get a # mechanism to prevent reconciling pages, using that on the first page of the initialn # fill should solve the problem.) # # However, because the cache size is fixed, the number of rows that the initial fill # generates can be used as an indicator: more rows mean that more updates were already # reconciled and there's less space to work with later. So, if we see enough rows that # there's not going to be any space for the later updates, skip the test on the grounds # that it's probably going to break. (Skip rather than fail because it's not wrong that # this happens; skip conditionally rather than disable the test because it does work an # appreciable fraction of the time and it's better to run it when possible.) # # I've picked an threshold based on some initial experiments. 141676 rows succeeds, # 143403 fails, so I picked 141677. Hopefully this will not need to be conditionalized # on the OS or machine type. # # Note that with 141676 rows there are several retries in the bottom loop, so things are # working as designed and the desired scenario is being tested. # While I'm pretty sure the above analysis is sound, the threshold is not as portable as # I'd hoped, so just skip the test entirely until someone has the patience to track down # a suitable threshold value for the test environment. #if self.valuefmt == '8t' and last_key >= 141677: # self.skipTest('Load too high; test will get stuck') if self.valuefmt == '8t': self.skipTest('Gets stuck and fails sometimes under load') # Now that the database contains as much data as will fit into # the configured cache, verify removes succeed. cursor = self.session.open_cursor(self.uri, None) for i in range(1, last_key // 4, 1): cursor.set_key(ds.key(i)) self.assertEqual(cursor.remove(), 0) cursor.reset() # Spin inserting to give eviction a chance to reclaim space sleeps = 0 inserted = False for i in range(1, 1000): try: cursor[ds.key(1)] = ds.value(1) except wiredtiger.WiredTigerError: cursor.reset() sleeps = sleeps + 1 self.assertLess(sleeps, 60 * 5) sleep(1) continue inserted = True break self.assertTrue(inserted)
def test_logts(self): # Create logged and non-logged objects. The non-logged objects are in two versions, one is # updated with a commit timestamp and one is not. Update the logged and non-logged timestamp # tables in a transaction with a commit timestamp and confirm the timestamps only apply to # the non-logged object. Update the non-logged, non-timestamp table in a transaction without # a commit timestamp, and confirm timestamps are ignored. uri_log = 'table:test_logts.log' ds_log = SimpleDataSet(self, uri_log, 100, key_format=self.key_format, value_format=self.value_format) ds_log.populate() c_log = self.session.open_cursor(uri_log) uri_ts = 'table:test_logts.ts' ds_ts = SimpleDataSet(self, uri_ts, 100, key_format=self.key_format, value_format=self.value_format, config='log=(enabled=false)') ds_ts.populate() c_ts = self.session.open_cursor(uri_ts) uri_nots = 'table:test_log04.nots' ds_nots = SimpleDataSet(self, uri_nots, 100, key_format=self.key_format, value_format=self.value_format, config='log=(enabled=false)') ds_nots.populate() c_nots = self.session.open_cursor(uri_nots) # Set oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) key = ds_ts.key(10) value10 = ds_ts.value(10) # Confirm initial data at timestamp 10. self.check(c_log, 10, key, value10) self.check(c_ts, 10, key, value10) self.check(c_nots, 10, key, value10) # Update and then rollback. value50 = ds_ts.value(50) self.session.begin_transaction() c_log[key] = value50 c_ts[key] = value50 c_nots[key] = value50 self.session.rollback_transaction() # Confirm data at time 10 and 20. self.check(c_log, 10, key, value10) self.check(c_ts, 10, key, value10) self.check(c_nots, 10, key, value10) self.check(c_log, 20, key, value10) self.check(c_ts, 20, key, value10) self.check(c_nots, 20, key, value10) # Update and then commit data at time 20. value55 = ds_ts.value(55) self.session.begin_transaction() c_log[key] = value55 c_ts[key] = value55 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20)) self.session.begin_transaction() c_nots[key] = value55 self.session.commit_transaction() # Confirm data at time 10 and 20. self.check(c_log, 10, key, value55) self.check(c_ts, 10, key, value10) self.check(c_nots, 10, key, value55) self.check(c_log, 20, key, value55) self.check(c_nots, 20, key, value55) # Update and then commit data at time 30. value60 = ds_ts.value(60) self.session.begin_transaction() c_log[key] = value60 c_ts[key] = value60 self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30)) self.session.begin_transaction() c_nots[key] = value60 self.session.commit_transaction() # Confirm data at time 20 and 30 self.check(c_log, 20, key, value60) self.check(c_ts, 20, key, value55) self.check(c_nots, 20, key, value60) self.check(c_log, 30, key, value60) self.check(c_ts, 30, key, value60) self.check(c_nots, 30, key, value60) # Move the stable timestamp to 25. Checkpoint and rollback to a timestamp. self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(25)) if self.ckpt: self.session.checkpoint() self.conn.rollback_to_stable() # Confirm data at time 20 and 30. self.check(c_log, 20, key, value60) self.check(c_ts, 20, key, value55) self.check(c_nots, 20, key, value60) self.check(c_log, 30, key, value60) self.check(c_ts, 30, key, value55) self.check(c_nots, 30, key, value60)
def test_truncate_simple(self): uri = self.type + self.name # layout: # the number of initial skipped records # the number of initial inserted records # the number of trailing skipped records # the number of trailing inserted records layout = [ # simple set of rows (0, 0, 0, 0), # trailing append list, no delete point overlap (0, 0, 0, self.skip - 3), # trailing append list, delete point overlap (0, 0, 0, self.skip + 3), # trailing skipped list, no delete point overlap (0, 0, self.skip - 3, 1), # trailing skipped list, delete point overlap (0, 0, self.skip + 3, 1), # leading insert list, no delete point overlap (0, self.skip - 3, 0, 0), # leading insert list, delete point overlap (0, self.skip + 3, 0, 0), # leading skipped list, no delete point overlap (self.skip - 3, 1, 0, 0), # leading skipped list, delete point overlap (self.skip + 3, 1, 0, 0), ] # list: truncation patterns applied on top of the layout. # # begin and end: -1 means pass None for the cursor arg to truncate. An # integer N, with 1 <= N < self.nentries, truncates from/to a cursor # positioned at that row. list = [ (-1, self.nentries), # begin to end, begin = None (1, -1), # begin to end, end = None (1, self.nentries), # begin to end (-1, self.nentries - self.skip), # begin to middle, begin = None (1, self.nentries - self.skip), # begin to middle (self.skip, -1), # middle to end, end = None (self.skip, self.nentries), # middle to end (self.skip, # middle to different middle self.nentries - self.skip), (1, 1), # begin to begin (self.nentries, self.nentries), # end to end (self.skip, self.skip) # middle to same middle ] # Using this data set to compare only, it doesn't create or populate. ds = SimpleDataSet(self, uri, 0, key_format=self.keyfmt, value_format=self.valuefmt, config=self.config) # Build the layout we're going to test total = self.nentries for begin_skipped,begin_insert,end_skipped,end_insert in layout: # skipped records require insert/append records if begin_skipped and not begin_insert or \ end_skipped and not end_insert: raise AssertionError('test error: skipped set without insert') for begin,end in list: ''' print '===== run:' print 'key:', self.keyfmt, 'begin:', begin, 'end:', end print 'total: ', total, \ 'begin_skipped:', begin_skipped, \ 'begin_insert:', begin_insert, \ 'end_skipped:', end_skipped, \ 'end_insert:', end_insert ''' # Build a dictionary of what the object should look like for # later comparison expected = {} # Create the object. self.session.create( uri, self.config + ',key_format=' + self.keyfmt + ',value_format=' + self.valuefmt) # Insert the records that aren't skipped or inserted. start = begin_skipped + begin_insert stop = self.nentries - (end_skipped + end_insert) cursor = self.session.open_cursor(uri, None) for i in range(start + 1, stop + 1): k = ds.key(i) v = ds.value(i) cursor[k] = v expected[k] = [v] cursor.close() # Optionally close and re-open the object to get a disk image # instead of a big insert list. if self.reopen: self.reopen_conn() # Optionally insert initial skipped records. cursor = self.session.open_cursor(uri, None, "overwrite") start = 0 for i in range(0, begin_skipped): start += 1 k = ds.key(start) expected[k] = [0] # Optionally insert leading records. for i in range(0, begin_insert): start += 1 k = ds.key(start) v = ds.value(start) cursor[k] = v expected[k] = [v] # Optionally insert trailing skipped records. for i in range(0, end_skipped): stop += 1 k = ds.key(stop) expected[k] = [0] # Optionally insert trailing records. for i in range(0, end_insert): stop += 1 k = ds.key(stop) v = ds.value(stop) cursor[k] = v expected[k] = [v] cursor.close() self.truncateRangeAndCheck(ds, uri, begin, end, expected) self.session.drop(uri, None)