class test_overwrite(wttest.WiredTigerTestCase): name = 'overwrite' scenarios = make_scenarios([ ('file-r', dict(type='file:', keyfmt='r')), ('file-S', dict(type='file:', keyfmt='S')), ('lsm-S', dict(type='lsm:', keyfmt='S')), ('table-r', dict(type='table:', keyfmt='r')), ('table-S', dict(type='table:', keyfmt='S')), ]) # Confirm a cursor configured with/without overwrite correctly handles # non-existent records during insert, remove and update operations. def test_overwrite_insert(self): uri = self.type + self.name simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) # Insert of an existing record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(key_populate(cursor, 5)) cursor.set_value('XXXXXXXXXX') self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) # One additional test for the insert method: duplicate the cursor with # overwrite configured and then the insert should succeed. This test # is only for the insert method because the remove and update method # failure modes are for non-existent records, and you cannot duplicate # cursor pointing to non-existent records. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(key_populate(cursor, 5)) dupc = self.session.open_cursor(None, cursor, "overwrite=true") dupc.set_value('XXXXXXXXXX') self.assertEquals(dupc.insert(), 0) # Insert of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(key_populate(cursor, 6)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(key_populate(cursor, 200)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(key_populate(cursor, 201)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.insert(), 0) def test_overwrite_remove(self): uri = self.type + self.name simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) # Remove of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(key_populate(cursor, 5)) self.assertEquals(cursor.remove(), 0) # Remove of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(key_populate(cursor, 6)) self.assertEquals(cursor.remove(), 0) # Remove of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(key_populate(cursor, 200)) self.assertEquals(cursor.remove(), wiredtiger.WT_NOTFOUND) # Remove of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(key_populate(cursor, 201)) self.assertEquals(cursor.remove(), 0) def test_overwrite_update(self): uri = self.type + self.name simple_populate(self, uri, 'key_format=' + self.keyfmt, 100) # Update of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(key_populate(cursor, 5)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), 0) # Update of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(key_populate(cursor, 6)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), 0) # Update of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(key_populate(cursor, 200)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), wiredtiger.WT_NOTFOUND) # Update of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(key_populate(cursor, 201)) cursor.set_value('XXXXXXXXXX') self.assertEquals(cursor.update(), 0)
class test_cursor_comparison(wttest.WiredTigerTestCase): name = 'test_compare' types = [('file', dict(type='file:', lsm=False, dataset=SimpleDataSet)), ('lsm', dict(type='table:', lsm=True, dataset=ComplexLSMDataSet)), ('table', dict(type='table:', lsm=False, dataset=ComplexDataSet))] keyfmt = [('integer', dict(keyfmt='i', valfmt='S')), ('recno', dict(keyfmt='r', valfmt='S')), ('recno-fix', dict(keyfmt='r', valfmt='8t')), ('string', dict(keyfmt='S', valfmt='S'))] # Discard invalid or unhelpful scenario combinations. def keep(name, d): if d['keyfmt'] == 'r': # Skip record number keys with LSM. if d['lsm']: return False # Skip complex data sets with FLCS. if d['valfmt'] == '8t' and d['dataset'] != SimpleDataSet: return False else: # Skip byte data with row-store. if d['valfmt'] == '8t': return False return True scenarios = make_scenarios(types, keyfmt, include=keep) def test_cursor_comparison(self): uri = self.type + 'compare' uriX = self.type + 'compareX' # Build the object. ds = self.dataset(self, uri, 100, key_format=self.keyfmt) dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt) ds.populate() dsX.populate() if self.type == 'file:': ix0_0 = None ix0_1 = None ix1_0 = None ixX_0 = None else: ix0_0 = self.session.open_cursor(ds.index_name(0), None) ix0_1 = self.session.open_cursor(ds.index_name(0), None) ix1_0 = self.session.open_cursor(ds.index_name(1), None) ixX_0 = self.session.open_cursor(dsX.index_name(0), None) ix0_0.next() ix0_1.next() ix1_0.next() ixX_0.next() c1 = self.session.open_cursor(uri, None) c2 = self.session.open_cursor(uri, None) # Confirm failure unless the keys are set. msg = '/requires key be set/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c1.compare(c2), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c2.compare(c1), msg) # Test cursors before they're positioned. c1.set_key(ds.key(10)) c2.set_key(ds.key(20)) self.assertGreater(c2.compare(c1), 0) self.assertLess(c1.compare(c2), 0) c2.set_key(ds.key(10)) self.assertEqual(c1.compare(c2), 0) self.assertEqual(c2.compare(c1), 0) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) cX.set_key(dsX.key(10)) msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cX.compare(c1), msg) msg = '/wt_cursor.* is None/' self.assertRaisesHavingMessage(RuntimeError, lambda: cX.compare(None), msg) if ix0_0 != None: self.assertEqual(ix0_0.compare(ix0_1), 0) ix0_1.reset() ix0_1.prev() self.assertLess(ix0_0.compare(ix0_1), 0) self.assertGreater(ix0_1.compare(ix0_0), 0) # Main table vs. index not allowed msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c1.compare(ix0_0), msg) # Two unrelated indices not allowed self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: ixX_0.compare(ix0_0), msg) # Two different indices from same table not allowed self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: ix0_0.compare(ix1_0), msg) # Test cursors after they're positioned (shouldn't matter for compare). c1.set_key(ds.key(10)) self.assertEqual(c1.search(), 0) c2.set_key(ds.key(20)) self.assertEqual(c2.search(), 0) self.assertGreater(c2.compare(c1), 0) self.assertLess(c1.compare(c2), 0) c2.set_key(ds.key(10)) self.assertEqual(c2.search(), 0) self.assertEqual(c1.compare(c2), 0) self.assertEqual(c2.compare(c1), 0) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) cX.set_key(dsX.key(10)) self.assertEqual(cX.search(), 0) msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cX.compare(c1), msg) def test_cursor_equality(self): uri = self.type + 'equality' uriX = self.type + 'compareX' # Build the object. ds = self.dataset(self, uri, 100, key_format=self.keyfmt) dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt) ds.populate() dsX.populate() if self.type == 'file:': ix0_0 = None ix0_1 = None ix1_0 = None ixX_0 = None else: ix0_0 = self.session.open_cursor(ds.index_name(0), None) ix0_1 = self.session.open_cursor(ds.index_name(0), None) ix1_0 = self.session.open_cursor(ds.index_name(1), None) ixX_0 = self.session.open_cursor(dsX.index_name(0), None) ix0_0.next() ix0_1.next() ix1_0.next() ixX_0.next() c1 = self.session.open_cursor(uri, None) c2 = self.session.open_cursor(uri, None) # Confirm failure unless the keys are set. msg = '/requires key be set/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c1.equals(c2), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c2.equals(c1), msg) # Test cursors before they're positioned. c1.set_key(ds.key(10)) c2.set_key(ds.key(20)) self.assertFalse(c1.equals(c2)) self.assertFalse(c2.equals(c1)) c2.set_key(ds.key(10)) self.assertTrue(c1.equals(c2)) self.assertTrue(c2.equals(c1)) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) cX.set_key(dsX.key(10)) msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cX.equals(c1), msg) msg = '/wt_cursor.* is None/' self.assertRaisesHavingMessage(RuntimeError, lambda: cX.equals(None), msg) if ix0_0 != None: self.assertTrue(ix0_0.equals(ix0_1)) ix0_1.reset() ix0_1.prev() self.assertFalse(ix0_0.equals(ix0_1)) # Main table vs. index not allowed msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c1.equals(ix0_0), msg) # Two unrelated indices not allowed self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: ixX_0.equals(ix0_0), msg) # Two different indices from same table not allowed self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: ix0_0.equals(ix1_0), msg) # Test cursors after they're positioned (internally, it's a different # search path if keys are positioned in the tree). c1.set_key(ds.key(10)) self.assertEqual(c1.search(), 0) c2.set_key(ds.key(20)) self.assertEqual(c2.search(), 0) self.assertFalse(c1.equals(c2)) self.assertFalse(c2.equals(c1)) c2.set_key(ds.key(10)) self.assertEqual(c2.search(), 0) self.assertTrue(c1.equals(c2)) self.assertTrue(c2.equals(c1)) # Confirm failure for different objects. cX = self.session.open_cursor(uriX, None) cX.set_key(dsX.key(10)) self.assertEqual(cX.search(), 0) msg = '/must reference the same object/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cX.equals(c1), msg)
class test_tiered14(wttest.WiredTigerTestCase): uri = "table:test_tiered14-{}" # format for subtests auth_token = "test_token" bucket = "mybucket" cachedir = "mybucket-cache" bucket_prefix = "pfx_" extension_name = "local_store" # FIXME-WT-7833: enable the commented scenarios and run the # test with the --long option. # The multiplier makes the size of keys and values progressively larger. # A multipler of 0 makes the keys and values a single length. multiplier = [ ('0', dict(multiplier=0)), ('S', dict(multiplier=1)), ('M', dict(multiplier=10)), #('L', dict(multiplier=100, long_only=True)), #('XL', dict(multiplier=1000, long_only=True)), ] keyfmt = [ ('integer', dict(keyfmt='i')), ('string', dict(keyfmt='S')), ] dataset = [ ('simple', dict(dataset='simple')), #('complex', dict(dataset='complex', long_only=True)), ] scenarios = wtscenario.make_scenarios(multiplier, keyfmt, dataset) def conn_config(self): if not os.path.exists(self.bucket): os.mkdir(self.bucket) return \ 'tiered_storage=(auth_token=%s,' % self.auth_token + \ 'bucket=%s,' % self.bucket + \ 'bucket_prefix=%s,' % self.bucket_prefix + \ 'cache_directory=%s,' % self.cachedir + \ 'name=%s),tiered_manager=(wait=0)' % self.extension_name # Load the local store extension. def conn_extensions(self, extlist): # Windows doesn't support dynamically loaded extension libraries. if os.name == 'nt': extlist.skip_if_missing = True extlist.extension('storage_sources', self.extension_name) def progress(self, s): outstr = "testnum {}, position {}: {}".format(self.testnum, self.position, s) self.verbose(3, outstr) self.pr(outstr) # Run a sequence of operations, indicated by a string. # a = add some number of keys # u = update some number of keys # c = checkpoint # r = reopen # f = flush_tier # . = check to make sure all expected values are present # # We require a unique test number so we get can generate a different uri from # previous runs. A different approach is to drop the uri, but then we need to # remove the bucket and cache, which is specific to the storage source extension. def playback(self, testnum, ops): self.testnum = testnum self.position = -1 uri = self.uri.format(testnum) self.progress('Running ops: {} using uri {}'.format(ops, uri)) if self.dataset == 'simple': ds = TrackedSimpleDataSet(self, uri, self.multiplier, key_format=self.keyfmt) elif self.dataset == 'complex': ds = TrackedComplexDataSet(self, uri, self.multiplier, key_format=self.keyfmt) # Populate for a tracked data set is needed to create the uri. ds.populate() inserted = 0 # At the end of the sequence of operations, do a final check ('.'). for op in ops + '.': self.position += 1 try: if op == 'f': self.progress('flush_tier') self.session.flush_tier(None) elif op == 'c': self.progress('checkpoint') self.session.checkpoint() elif op == 'r': self.progress('reopen') self.reopen_conn() elif op == 'a': self.progress('add') n = random.randrange(1, 101) # 1 <= n <= 100 ds.store_range(inserted, n) inserted += n elif op == 'u': self.progress('update') # only update the elements if enough have already been added. n = random.randrange(1, 101) # 1 <= n <= 100 if n < inserted: pos = random.randrange(0, inserted - n) ds.store_range(pos, n) elif op == '.': self.progress('check') ds.check() except Exception as e: self.progress('Failed at position {} in {}: {}'.format( idx, ops, str(e))) raise (e) # Test tiered storage with checkpoints and flush_tier calls. def test_tiered(self): random.seed(0) # Get started with a fixed sequence of basic operations. # There's no particular reason to start with this sequence. testnum = 0 self.playback(testnum, "aaaaacaaa.uucrauaf.aauaac.auu.aacrauafa.uruua.") for i in range(0, 10): testnum += 1 # Generate a sequence of 100 operations that is heavy on additions and updates. s = ''.join(random.choices('aaaaauuuuufcr.', k=100)) self.playback(testnum, s) for i in range(0, 10): testnum += 1 # Generate a sequence of 100 operations that is has a greater mix of 'operational' functions. s = ''.join(random.choices('aufcr.', k=100)) self.playback(testnum, s)
class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): logmax = "100K" tablename1 = 'test_cursor07_log' tablename2 = 'test_cursor07_nolog' tablename3 = 'test_cursor07_nologtxn' uri1 = 'table:' + tablename1 uri2 = 'table:' + tablename2 uri3 = 'table:' + tablename3 # A large number of keys will force a log file change which will # test that scenario for log cursors. nkeys = 7000 scenarios = make_scenarios([('regular', dict(reopen=False)), ('reopen', dict(reopen=True))]) # Enable logging for this test. def conn_config(self): return 'log=(enabled,file_max=%s,remove=false),' % self.logmax + \ 'transaction_sync="(method=dsync,enabled)"' def test_log_cursor(self): # print "Creating %s with config '%s'" % (self.uri, self.create_params) create_params = 'key_format=i,value_format=u' create_nolog_params = 'key_format=i,value_format=u,log=(enabled=false)' self.session.create(self.uri1, create_params) c1 = self.session.open_cursor(self.uri1, None) self.session.create(self.uri2, create_nolog_params) c2 = self.session.open_cursor(self.uri2, None) self.session.create(self.uri3, create_nolog_params) c3 = self.session.open_cursor(self.uri3, None) # A binary value. value = b'\x01\x02abcd\x03\x04' value_nolog = b'\x01\x02dcba\x03\x04' # We want to test both adding data to a table that is not logged # that is part of the same transaction as a table that is logged # as well as in its own transaction. self.session.begin_transaction() for k in range(self.nkeys): c1[k] = value c3[k] = value_nolog self.session.commit_transaction() c1.close() c3.close() self.session.begin_transaction() for k in range(self.nkeys): c2[k] = value_nolog self.session.commit_transaction() c2.close() if self.reopen: self.reopen_conn() # Check for these values via a log cursor c = self.session.open_cursor("log:", None) count = 0 while c.next() == 0: # lsn.file, lsn.offset, opcount keys = c.get_key() # txnid, rectype, optype, fileid, logrec_key, logrec_value values = c.get_value() # We are only looking for log records that that have a key/value # pair. if values[4] != b'': if value in values[5]: # logrec_value count += 1 c.close() self.assertEqual(count, self.nkeys)
class test_cursor_random(wttest.WiredTigerTestCase): types = [('file', dict(type='file:random', dataset=SimpleDataSet)), ('table', dict(type='table:random', dataset=ComplexDataSet))] config = [('sample', dict(config='next_random=true,next_random_sample_size=35')), ('not-sample', dict(config='next_random=true'))] scenarios = make_scenarios(types, config) # Check that opening a random cursor on a row-store returns not-supported # for methods other than next, reconfigure and reset, and next returns # not-found. def test_cursor_random(self): uri = self.type self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None, self.config) msg = "/Unsupported cursor/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.compare(cursor), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.insert(), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.prev(), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.remove(), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.search(), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.search_near(), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.update(), msg) self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) self.assertEquals(cursor.reconfigure(), 0) self.assertEquals(cursor.reset(), 0) cursor.close() # Check that next_random works with a single value, repeatedly. def test_cursor_random_single_record(self): uri = self.type self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None) cursor['AAA'] = 'BBB' cursor.close() cursor = self.session.open_cursor(uri, None, self.config) for i in range(1, 5): self.assertEquals(cursor.next(), 0) self.assertEquals(cursor.get_key(), 'AAA') cursor.close # Check that next_random works in the presence of a larger set of values, # where the values are in an insert list. def test_cursor_random_multiple_insert_records(self): uri = self.type ds = self.dataset(self, uri, 100, config='allocation_size=512,leaf_page_max=512') ds.populate() # In a insert list, next_random always selects the middle key/value # pair, all we can do is confirm cursor.next works. cursor = self.session.open_cursor(uri, None, self.config) self.assertEqual(cursor.next(), 0) # Check that next_random works in the presence of a larger set of values, # where the values are in a disk format page. def cursor_random_multiple_page_records(self, reopen): uri = self.type ds = self.dataset(self, uri, 10000, config='allocation_size=512,leaf_page_max=512') ds.populate() # Optionally close the connection so everything is forced to disk, # insert lists are an entirely different path in the code. if reopen: self.reopen_conn() cursor = self.session.open_cursor(uri, None, self.config) last = '' match = 0 for i in range(1, 10): self.assertEqual(cursor.next(), 0) current = cursor.get_key() if current == last: match += 1 last = current self.assertLess( match, 5, 'next_random did not return random records, too many matches found' ) def test_cursor_random_multiple_page_records_reopen(self): self.cursor_random_multiple_page_records(1) def test_cursor_random_multiple_page_records(self): self.cursor_random_multiple_page_records(0)
class test_stat01(wttest.WiredTigerTestCase): """ Test statistics """ config = 'internal_page_max=4K,leaf_page_max=8K' nentries = 25 types = [('file', dict(uri='file:test_stat01.wt')), ('table', dict(uri='table:test_stat01.wt'))] keyfmt = [ ('recno', dict(keyfmt='r')), ('string', dict(keyfmt='S')), ] scenarios = make_scenarios(types, keyfmt) conn_config = 'statistics=(all)' def statstr_to_int(self, str): """ Convert a statistics value string, which may be in either form: '12345' or '33M (33604836)' """ parts = str.rpartition('(') return int(parts[2].rstrip(')')) # Do a quick check of the entries in the the stats cursor, the "lookfor" # string should appear with a minimum value of least "min". def check_stats(self, statcursor, min, lookfor): stringclass = ''.__class__ intclass = (0).__class__ # Reset the cursor, we're called multiple times. statcursor.reset() found = False foundval = 0 for id, desc, valstr, val in statcursor: self.assertEqual(type(desc), stringclass) self.assertEqual(type(valstr), stringclass) self.assertEqual(type(val), intclass) self.assertEqual(val, self.statstr_to_int(valstr)) self.printVerbose( 2, ' stat: \'' + desc + '\', \'' + valstr + '\', ' + str(val)) if desc == lookfor: found = True foundval = val self.assertTrue(found, 'in stats, did not see: ' + lookfor) self.assertTrue(foundval >= min) # Test simple connection statistics. def test_basic_conn_stats(self): # Build an object and force some writes. SimpleDataSet(self, self.uri, 1000, config=self.config, key_format=self.keyfmt).populate() self.session.checkpoint(None) # See that we can get a specific stat value by its key and verify its # entry is self-consistent. allstat_cursor = self.session.open_cursor('statistics:', None, None) self.check_stats(allstat_cursor, 10, 'block-manager: blocks written') values = allstat_cursor[stat.conn.block_write] self.assertEqual(values[0], 'block-manager: blocks written') val = self.statstr_to_int(values[1]) self.assertEqual(val, values[2]) allstat_cursor.close() # Test simple object statistics. def test_basic_data_source_stats(self): # Build an object. config = self.config + ',key_format=' + self.keyfmt self.session.create(self.uri, config) cursor = self.session.open_cursor(self.uri, None, None) value = "" for i in range(1, self.nentries): value = value + 1000 * "a" cursor[simple_key(cursor, i)] = value cursor.close() # Force the object to disk, otherwise we can't check the overflow count. self.reopen_conn() # See that we can get a specific stat value by its key and verify its # entry is self-consistent. cursor = self.session.open_cursor('statistics:' + self.uri, None, None) self.check_stats(cursor, 8192, 'btree: maximum leaf page size') self.check_stats(cursor, 4096, 'btree: maximum internal page size') self.check_stats(cursor, 10, 'btree: overflow pages') values = cursor[stat.dsrc.btree_overflow] self.assertEqual(values[0], 'btree: overflow pages') val = self.statstr_to_int(values[1]) self.assertEqual(val, values[2]) cursor.close() cursor = self.session.open_cursor('statistics:' + self.uri, None, "statistics=(size)") values = cursor[stat.dsrc.block_size] self.assertNotEqual(values[2], 0) cursor.close() # Test simple per-checkpoint statistics. def test_checkpoint_stats(self): ds = SimpleDataSet(self, self.uri, self.nentries, config=self.config, key_format=self.keyfmt) for name in ('first', 'second', 'third'): ds.populate() self.session.checkpoint('name=' + name) cursor = self.session.open_cursor('statistics:' + self.uri, None, 'checkpoint=' + name) self.assertEqual(cursor[stat.dsrc.btree_entries][2], self.nentries) cursor.close() def test_missing_file_stats(self): self.assertRaises( wiredtiger.WiredTigerError, lambda: self.session.open_cursor('statistics:file:DoesNotExist'))
class test_rollback_to_stable19(test_rollback_to_stable_base): session_config = 'isolation=snapshot' in_memory_values = [('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True))] key_format_values = [ ('column', dict(key_format='r')), ('integer_row', dict(key_format='i')), ] restart_options = [ ('shutdown', dict(crash='false')), ('crash', dict(crash='true')), ] scenarios = make_scenarios(in_memory_values, key_format_values, restart_options) def conn_config(self): config = 'cache_size=50MB,statistics=(all),log=(enabled=false),eviction_dirty_trigger=5,eviction_updates_trigger=5' if self.in_memory: config += ',in_memory=true' else: config += ',in_memory=false' return config def test_rollback_to_stable_no_history(self): nrows = 1000 # Prepare transactions for column store table is not yet supported. if self.key_format == 'r': self.skipTest( 'Prepare transactions for column store table is not yet supported' ) # Create a table without logging. uri = "table:rollback_to_stable19" ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + ',stable_timestamp=' + timestamp_str(10)) valuea = "aaaaa" * 100 # Perform several updates and removes. s = self.conn.open_session() cursor = s.open_cursor(uri) s.begin_transaction() for i in range(1, nrows + 1): cursor[ds.key(i)] = valuea cursor.set_key(i) cursor.remove() cursor.close() s.prepare_transaction('prepare_timestamp=' + timestamp_str(20)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") evict_cursor.set_key(1) evict_cursor.search() evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() # Pin stable timestamp to 20. self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20)) if not self.in_memory: self.session.checkpoint() if not self.in_memory: if self.crash: simulate_crash_restart(self, ".", "RESTART") else: # Close and reopen the connection self.reopen_conn() else: self.conn.rollback_to_stable() s.rollback_transaction() # Verify data is not visible. self.check(valuea, uri, 0, 20) self.check(valuea, uri, 0, 30) stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] self.assertGreater(upd_aborted, 0) self.assertGreater(keys_removed, 0) def test_rollback_to_stable_with_history(self): nrows = 1000 # Prepare transactions for column store table is not yet supported. if self.key_format == 'r': self.skipTest( 'Prepare transactions for column store table is not yet supported' ) # Create a table without logging. uri = "table:rollback_to_stable19" ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable timestamps to 10. self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + ',stable_timestamp=' + timestamp_str(10)) valuea = "aaaaa" * 100 valueb = "bbbbb" * 100 # Perform several updates. self.large_updates(uri, valuea, ds, nrows, 0, 20) # Perform several removes. self.large_removes(uri, ds, nrows, 0, 30) # Perform several updates and removes. s = self.conn.open_session() cursor = s.open_cursor(uri) s.begin_transaction() for i in range(1, nrows + 1): cursor[ds.key(i)] = valueb cursor.set_key(i) cursor.remove() cursor.close() s.prepare_transaction('prepare_timestamp=' + timestamp_str(40)) # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") # Search for the key so we position our cursor on the page that we want to evict. self.session.begin_transaction("ignore_prepare = true") evict_cursor.set_key(1) evict_cursor.search() evict_cursor.reset() evict_cursor.close() self.session.commit_transaction() # Pin stable timestamp to 40. self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40)) if not self.in_memory: self.session.checkpoint() if not self.in_memory: if self.crash: simulate_crash_restart(self, ".", "RESTART") else: # Close and reopen the connection self.reopen_conn() else: self.conn.rollback_to_stable() s.rollback_transaction() # Verify data. self.check(valuea, uri, nrows, 20) self.check(valuea, uri, 0, 30) self.check(valuea, uri, 0, 40) stat_cursor = self.session.open_cursor('statistics:', None, None) upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] self.assertGreater(upd_aborted, 0) if not self.in_memory: self.assertGreater(hs_removed, 0)
class test_config03(test_base03.test_base03): K = 1024 M = 1024 * K G = 1024 * M cache_size_scenarios = wtscenario.quick_scenarios( 's_cache_size', [1 * M, 20 * M, 100 * M, 1 * G, None], [0.6, 0.6, 0.6, 0.6, 0.6]) create_scenarios = wtscenario.quick_scenarios('s_create', [True, False, None], [1.0, 0.2, 0.3]) error_prefix_scenarios = wtscenario.quick_scenarios( 's_error_prefix', [None, "errpfx:"], [1.0, 0.2]) # eviction_target < eviction_trigger -- checked later eviction_target_scenarios = wtscenario.quick_scenarios( 's_eviction_target', [10, 40, 85, 98], None) eviction_trigger_scenarios = wtscenario.quick_scenarios( 's_eviction_trigger', [50, 90, 95, 99], None) multiprocess_scenarios = wtscenario.quick_scenarios( 's_multiprocess', [True, False], [1.0, 1.0]) session_max_scenarios = wtscenario.quick_scenarios('s_session_max', [3, 30, 300], None) transactional_scenarios = wtscenario.quick_scenarios( 's_transactional', [True, False], [0.2, 1.0]) # Note: we are not using any truly verbose scenarios until we have # a way to redirect verbose output to a file in Python. # #verbose_scenarios = wtscenario.quick_scenarios('s_verbose', # ['block', 'evict,evictserver', 'fileops,hazard,mutex', # 'read,readserver,reconcile,salvage','verify,write',''], None) verbose_scenarios = wtscenario.quick_scenarios('s_verbose', [None], None) config_vars = [ 'cache_size', 'create', 'error_prefix', 'eviction_target', 'eviction_trigger', 'multiprocess', 'session_max', 'verbose' ] scenarios = wtscenario.make_scenarios(cache_size_scenarios, create_scenarios, error_prefix_scenarios, eviction_target_scenarios, eviction_trigger_scenarios, multiprocess_scenarios, session_max_scenarios, transactional_scenarios, verbose_scenarios, prune=100, prunelong=1000) #wttest.WiredTigerTestCase.printVerbose(2, 'test_config03: running ' + \ # str(len(scenarios)) + ' of ' + \ # str(len(all_scenarios)) + ' possible scenarios') def setUpConnectionOpen(self, dir): args = '' # add names to args, e.g. args += ',session_max=30' for var in self.config_vars: value = getattr(self, 's_' + var) if value != None: if var == 'verbose': value = '[' + str(value) + ']' if value == True: value = 'true' if value == False: value = 'false' args += ',' + var + '=' + str(value) args += ',' self.pr('wiredtiger_open with args: ' + args) expect_fail = False successargs = args if self.s_create == False: successargs = successargs.replace(',create=false,', ',create,') expect_fail = True fail_msg = '/(No such file or directory|The system cannot find the file specified)/' elif self.s_create == None: successargs = successargs + 'create=true,' expect_fail = True fail_msg = '/(No such file or directory|The system cannot find the file specified)/' if self.s_eviction_target >= self.s_eviction_trigger: # construct args that guarantee that target < trigger # we know that trigger >= 1 repfrom = ',eviction_target=' + str(self.s_eviction_target) repto = ',eviction_target=' + str(self.s_eviction_trigger - 1) successargs = successargs.replace(repfrom, repto) if not expect_fail: expect_fail = True fail_msg = \ '/eviction target must be lower than the eviction trigger/' if expect_fail: self.verbose(3, 'wiredtiger_open (should fail) with args: ' + args) self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: wiredtiger.wiredtiger_open(dir, args), fail_msg) args = successargs self.verbose(3, 'wiredtiger_open with args: ' + args) conn = self.wiredtiger_open(dir, args) self.pr( ` conn `) return conn
class test_checkpoint(wttest.WiredTigerTestCase): conn_config = 'statistics=(all),timing_stress_for_test=[checkpoint_slow]' session_config = 'isolation=snapshot' format_values = [ ('column-fix', dict(key_format='r', value_format='8t', extraconfig=',allocation_size=512,leaf_page_max=512')), ('column', dict(key_format='r', value_format='S', extraconfig='')), ('string_row', dict(key_format='S', value_format='S', extraconfig='')), ] name_values = [ ('nn', dict(first_checkpoint='first_checkpoint', second_checkpoint='second_checkpoint')), ('nu', dict(first_checkpoint='first_checkpoint', second_checkpoint=None)), # This doesn't work because there's no way to open the first unnamed checkpoint. #('un', dict(first_checkpoint=None, second_checkpoint='second_checkpoint')), ] scenarios = make_scenarios(format_values, name_values) def large_updates(self, uri, ds, nrows, value): cursor = self.session.open_cursor(uri) self.session.begin_transaction() for i in range(1, nrows + 1): cursor[ds.key(i)] = value if i % 101 == 0: self.session.commit_transaction() self.session.begin_transaction() self.session.commit_transaction() cursor.close() # "expected" is a list of maps from values to counts of values. def check(self, ds, ckpt, nrows, value): if ckpt is None: ckpt = 'WiredTigerCheckpoint' cursor = self.session.open_cursor(ds.uri, None, 'checkpoint=' + ckpt) #self.session.begin_transaction() count = 0 for k, v in cursor: self.assertEqual(v, value) count += 1 self.assertEqual(count, nrows) #self.session.rollback_transaction() cursor.close() def test_checkpoint(self): uri = 'table:checkpoint14' nrows = 10000 # Create a table. ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format, config=self.extraconfig) ds.populate() if self.value_format == '8t': nrows *= 5 value_a = 97 value_b = 98 value_c = 99 else: value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 value_c = "ccccc" * 100 # Write some baseline data. self.large_updates(uri, ds, nrows, value_a) # Write this data out now so we aren't waiting for it while trying to # race with the later data. self.session.checkpoint() # Write some more data, and hold the transaction open. session2 = self.conn.open_session() cursor2 = session2.open_cursor(uri) session2.begin_transaction() for i in range(1, nrows + 1): cursor2[ds.key(i)] = value_b # Checkpoint in the background. done = threading.Event() if self.first_checkpoint is None: ckpt = checkpoint_thread(self.conn, done) else: ckpt = named_checkpoint_thread(self.conn, done, self.first_checkpoint) try: ckpt.start() # Wait for checkpoint to start before committing. ckpt_started = 0 while not ckpt_started: stat_cursor = self.session.open_cursor('statistics:', None, None) ckpt_started = stat_cursor[stat.conn.txn_checkpoint_running][2] stat_cursor.close() time.sleep(1) session2.commit_transaction() finally: done.set() ckpt.join() # Rinse and repeat. session2.begin_transaction() for i in range(1, nrows + 1): cursor2[ds.key(i)] = value_c # Checkpoint in the background. done = threading.Event() if self.second_checkpoint is None: ckpt = checkpoint_thread(self.conn, done) else: ckpt = named_checkpoint_thread(self.conn, done, self.second_checkpoint) try: ckpt.start() # Sleep a bit so that checkpoint starts before committing last transaction. time.sleep(2) session2.commit_transaction() finally: done.set() ckpt.join() # Other tests check for whether the visibility of a partially-written transaction # is handled correctly. Here we're interested in whether the visibility mechanism # is using the right snapshot for the checkpoint we're reading. So insist that we # not see the value_b transaction in the first checkpoint, or the value_c transaction # in the second checkpoint. If test machine lag causes either transaction to commit # before the checkpoint starts, we'll see value_b in the first checkpoint and/or # value_c in the second. But also, if we end up using the second checkpoint's snapshot # for the first checkpoint, we'll see value_b. So if this happens more than once in a # blue moon we should probably strengthen the test so we can more reliably distinguish # the cases, probably by doing a third transaction/checkpoint pair. # # If we end up using the first checkpoint's snapshot for reading the second checkpoint, # we'll most likely see no data at all; that would be a serious failure if it happened. # Read the checkpoints. self.check(ds, self.first_checkpoint, nrows, value_a) self.check(ds, self.second_checkpoint, nrows, value_b) # If we haven't died yet, pretend to crash, and run RTS to see if the # (second) checkpoint was inconsistent. Unfortunately we can't readily # check on both. simulate_crash_restart(self, ".", "RESTART") # Make sure we did get an inconsistent checkpoint. stat_cursor = self.session.open_cursor('statistics:', None, None) inconsistent_ckpt = stat_cursor[stat.conn.txn_rts_inconsistent_ckpt][2] stat_cursor.close() self.assertGreater(inconsistent_ckpt, 0)
class test_rollback_to_stable06(test_rollback_to_stable_base): session_config = 'isolation=snapshot' in_memory_values = [('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True))] prepare_values = [('no_prepare', dict(prepare=False)), ('prepare', dict(prepare=True))] scenarios = make_scenarios(in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' if self.in_memory: config += ',in_memory=true' else: config += ',log=(enabled),in_memory=false' return config def test_rollback_to_stable(self): nrows = 1000 # Create a table without logging. uri = "table:rollback_to_stable06" ds = SimpleDataSet(self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) + ',stable_timestamp=' + timestamp_str(10)) value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 value_c = "ccccc" * 100 value_d = "ddddd" * 100 # Perform several updates. self.large_updates(uri, value_a, ds, nrows, 20) self.large_updates(uri, value_b, ds, nrows, 30) self.large_updates(uri, value_c, ds, nrows, 40) self.large_updates(uri, value_d, ds, nrows, 50) # Verify data is visible and correct. self.check(value_a, uri, nrows, 20) self.check(value_b, uri, nrows, 30) self.check(value_c, uri, nrows, 40) self.check(value_d, uri, nrows, 50) # Checkpoint to ensure the data is flushed, then rollback to the stable timestamp. if not self.in_memory: self.session.checkpoint() self.conn.rollback_to_stable() # Check that all keys are removed. self.check(value_a, uri, 0, 20) self.check(value_b, uri, 0, 30) self.check(value_c, uri, 0, 40) self.check(value_d, uri, 0, 50) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] stat_cursor.close() self.assertEqual(calls, 1) self.assertEqual(keys_restored, 0) self.assertGreater(pages_visited, 0) self.assertGreaterEqual(keys_removed, 0) if self.in_memory: self.assertEqual(upd_aborted, nrows * 4) self.assertEqual(hs_removed, 0) else: self.assertGreaterEqual(upd_aborted, 0) self.assertGreaterEqual(hs_removed, nrows * 3)
class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'ts07_ts_nologged' tablename2 = 'ts07_nots_logged' tablename3 = 'ts07_ts_logged' format_values = [ ('string-row', dict(key_format='i', value_format='S')), ('column', dict(key_format='r', value_format='S')), ('column-fix', dict(key_format='r', value_format='8t')), ] types = [ ('file', dict(uri='file:', use_cg=False, use_index=False)), ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), ] conncfg = [ ('nolog', dict(conn_config='create,cache_size=2M', using_log=False)), ('log', dict(conn_config='create,log=(file_max=1M,archive=false,enabled),cache_size=2M', using_log=True)), ] nkeys = [ ('100keys', dict(nkeys=100)), ('500keys', dict(nkeys=500)), ('1000keys', dict(nkeys=1000)), ] scenarios = make_scenarios(format_values, types, conncfg, nkeys) # Binary values. def moreinit(self): if self.value_format == '8t': self.value = 2 self.value2 = 4 self.value3 = 6 else: self.value = u'\u0001\u0002abcd\u0007\u0004' self.value2 = u'\u0001\u0002dcba\u0007\u0004' self.value3 = u'\u0001\u0002cdef\u0007\u0004' # Check that a cursor (optionally started in a new transaction), sees the # expected value for a key def check(self, session, txn_config, k, expected, flcs_expected): # In FLCS the table extends under uncommitted writes and we expect to # see zero rather than NOTFOUND. if self.value_format == '8t' and flcs_expected is not None: expected = flcs_expected if txn_config: session.begin_transaction(txn_config) c = session.open_cursor(self.uri + self.tablename, None) if expected is None: c.set_key(k) self.assertEqual(c.search(), wiredtiger.WT_NOTFOUND) else: self.assertEqual(c[k], expected) c.close() if txn_config: session.commit_transaction() # Check reads of all tables at a timestamp def check_reads(self, session, txn_config, check_value, valcnt, valcnt2, valcnt3): if txn_config: session.begin_transaction(txn_config) c = session.open_cursor(self.uri + self.tablename, None) c2 = session.open_cursor(self.uri + self.tablename2, None) c3 = session.open_cursor(self.uri + self.tablename3, None) # In FLCS the values are bytes, which are numbers, but the tests below are via # string inclusion rather than just equality of values. Not sure why that is, but # I'm going to assume there's a reason for it and not change things. Compensate. if self.value_format == '8t': check_value = str(check_value) count = 0 for k, v in c: if check_value in str(v): count += 1 c.close() count2 = 0 for k, v in c2: if check_value in str(v): count2 += 1 c2.close() count3 = 0 for k, v in c3: if check_value in str(v): count3 += 1 c3.close() if txn_config: session.commit_transaction() self.assertEqual(count, valcnt) self.assertEqual(count2, valcnt2) self.assertEqual(count3, valcnt3) # # Take a backup of the database and verify that the value we want to # check exists in the tables the expected number of times. # def backup_check(self, check_value, valcnt, valcnt2, valcnt3): newdir = "BACKUP" copy_wiredtiger_home(self, '.', newdir, True) conn = self.setUpConnectionOpen(newdir) session = self.setUpSessionOpen(conn) c = session.open_cursor(self.uri + self.tablename, None) c2 = session.open_cursor(self.uri + self.tablename2, None) c3 = session.open_cursor(self.uri + self.tablename3, None) # In FLCS the values are bytes, which are numbers, but the tests below are via # string inclusion rather than just equality of values. Not sure why that is, but # I'm going to assume there's a reason for it and not change things. Compensate. if self.value_format == '8t': check_value = str(check_value) # Count how many times the second value is present count = 0 for k, v in c: if check_value in str(v): # print "check_value found in key " + str(k) count += 1 c.close() # Count how many times the second value is present in the # non-timestamp table. count2 = 0 for k, v in c2: if check_value in str(v): # print "check_value found in key " + str(k) count2 += 1 c2.close() # Count how many times the second value is present in the # logged timestamp table. count3 = 0 for k, v in c3: if check_value in str(v): count3 += 1 c3.close() conn.close() # print "CHECK BACKUP: Count " + str(count) + " Count2 " + str(count2) + " Count3 " + str(count3) # print "CHECK BACKUP: Expect value2 count " + str(valcnt) # print "CHECK BACKUP: 2nd table Expect value2 count " + str(valcnt2) # print "CHECK BACKUP: 3rd table Expect value2 count " + str(valcnt3) self.assertEqual(count, valcnt) self.assertEqual(count2, valcnt2) self.assertEqual(count3, valcnt3) # Check that a cursor sees the expected values after a checkpoint. def ckpt_backup(self, check_value, valcnt, valcnt2, valcnt3): # Take a checkpoint. Make a copy of the database. Open the # copy and verify whether or not the expected data is in there. ckptcfg = 'use_timestamp=true' self.session.checkpoint(ckptcfg) self.backup_check(check_value, valcnt, valcnt2, valcnt3) def check_stable(self, check_value, valcnt, valcnt2, valcnt3): self.ckpt_backup(check_value, valcnt, valcnt2, valcnt3) # When reading as-of a timestamp, tables 1 and 3 should match (both # use timestamps and we're not running recovery, so logging behavior # should be irrelevant). self.check_reads(self.session, 'read_timestamp=' + self.stablets, check_value, valcnt, valcnt2, valcnt) def test_timestamp07(self): uri = self.uri + self.tablename uri2 = self.uri + self.tablename2 uri3 = self.uri + self.tablename3 self.moreinit() # # Open three tables: # 1. Table is not logged and uses timestamps. # 2. Table is logged and does not use timestamps. # 3. Table is logged and uses timestamps. # format = 'key_format={},value_format={}'.format(self.key_format, self.value_format) self.session.create(uri, format + ',log=(enabled=false)') c = self.session.open_cursor(uri) self.session.create(uri2, format) c2 = self.session.open_cursor(uri2) self.session.create(uri3, format) c3 = self.session.open_cursor(uri3) # print "tables created" # Insert keys 1..nkeys each with timestamp=key, in some order. orig_keys = list(range(1, self.nkeys+1)) keys = orig_keys[:] random.shuffle(keys) for k in keys: c2[k] = self.value self.session.begin_transaction() c[k] = self.value c3[k] = self.value self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(k)) # print "value inserted in all tables, reading..." # Now check that we see the expected state when reading at each # timestamp. for k in orig_keys: self.check(self.session, 'read_timestamp=' + self.timestamp_str(k), k, self.value, None) self.check(self.session, 'read_timestamp=' + self.timestamp_str(k), k + 1, None, None if k == self.nkeys else 0) # print "all values read, updating timestamps" # Bump the oldest timestamp, we're not going back... self.assertTimestampsEqual(self.conn.query_timestamp(), self.timestamp_str(self.nkeys)) self.oldts = self.stablets = self.timestamp_str(self.nkeys) self.conn.set_timestamp('oldest_timestamp=' + self.oldts) self.conn.set_timestamp('stable_timestamp=' + self.stablets) # print "Oldest " + self.oldts # print "inserting value2 in all tables" # Update them and retry. random.shuffle(keys) count = 0 for k in keys: # Make sure a timestamp cursor is the last one to update. This # tests the scenario for a bug we found where recovery replayed # the last record written into the log. # # print "Key " + str(k) + " to value2" c2[k] = self.value2 self.session.begin_transaction() c[k] = self.value2 c3[k] = self.value2 ts = self.timestamp_str(k + self.nkeys) self.session.commit_transaction('commit_timestamp=' + ts) # print "Commit key " + str(k) + " ts " + ts count += 1 # print "Updated " + str(count) + " keys to value2" # Take a checkpoint using the given configuration. Then verify # whether value2 appears in a copy of that data or not. # print "check_stable 1" self.check_stable(self.value2, 0, self.nkeys, self.nkeys if self.using_log else 0) # Update the stable timestamp to the latest, but not the oldest # timestamp and make sure we can see the data. Once the stable # timestamp is moved we should see all keys with value2. self.stablets = self.timestamp_str(self.nkeys*2) self.conn.set_timestamp('stable_timestamp=' + self.stablets) # print "check_stable 2" self.check_stable(self.value2, self.nkeys, self.nkeys, self.nkeys) # If we're not using the log we're done. if not self.using_log: return # Update the key and retry. This time take a backup and recover. random.shuffle(keys) count = 0 for k in keys: # Make sure a timestamp cursor is the last one to update. This # tests the scenario for a bug we found where recovery replayed # the last record written into the log. # # print "Key " + str(k) + " to value3" c2[k] = self.value3 self.session.begin_transaction() c[k] = self.value3 c3[k] = self.value3 ts = self.timestamp_str(k + self.nkeys*2) self.session.commit_transaction('commit_timestamp=' + ts) # print "Commit key " + str(k) + " ts " + ts count += 1 # print "Updated " + str(count) + " keys to value3" # Flush the log but don't checkpoint self.session.log_flush('sync=on') # Take a backup and then verify whether value3 appears in a copy # of that data or not. Both tables that are logged should see # all the data regardless of timestamps. The table that is not # logged should not see any of it. # print "check_stable 3" self.check_stable(self.value3, 0, self.nkeys, self.nkeys)
class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_readonly01' create = True entries = 10000 # # We want a list of directory writable or readonly. # basecfg_list = [ ('basecfg', dict(basecfg='config_base=true,')), ('no_basecfg', dict(basecfg='config_base=false,')), ] dir_list = [ ('write', dict(dirchmod=False)), ('readonly', dict(dirchmod=True)), ] log_list = [ ('logging', dict(logcfg='log=(archive=false,enabled,file_max=100K),')), ('no_logging', dict(logcfg='log=(enabled=false),')), ] types = [ ('lsm', dict(tabletype='lsm', uri='lsm', create_params='key_format=i,value_format=i')), ('file-row', dict(tabletype='row', uri='file', create_params='key_format=i,value_format=i')), ('file-var', dict(tabletype='var', uri='file', create_params='key_format=r,value_format=i')), ('file-fix', dict(tabletype='fix', uri='file', create_params='key_format=r,value_format=8t')), ('table-row', dict(tabletype='row', uri='table', create_params='key_format=i,value_format=i')), ('table-var', dict(tabletype='var', uri='table', create_params='key_format=r,value_format=i')), ('table-fix', dict(tabletype='fix', uri='table', create_params='key_format=r,value_format=8t')), ] scenarios = make_scenarios(basecfg_list, dir_list, log_list, types) def conn_config(self): params = \ 'error_prefix="%s",' % self.shortid() + \ '%s' % self.logcfg + \ '%s' % self.basecfg if self.create: conn_params = 'create,' + params else: conn_params = 'readonly=true,' + params return conn_params def close_reopen(self): ''' Close the connection and reopen readonly''' # # close the original connection. If needed, chmod the # database directory to readonly mode. Then reopen the # connection with readonly. # self.close_conn() # # The chmod command is not fully portable to windows. # if self.dirchmod and os.name == 'posix': for f in os.listdir(self.home): if os.path.isfile(f): os.chmod(f, 0444) os.chmod(self.home, 0555) self.conn = self.setUpConnectionOpen(self.home) self.session = self.setUpSessionOpen(self.conn) def readonly(self): # Here's the strategy: # - Create a table. # - Insert data into table. # - Close connection. # - Possibly chmod to readonly # - Open connection readonly # - Confirm we can read the data. # tablearg = self.uri + ':' + self.tablename self.session.create(tablearg, self.create_params) c = self.session.open_cursor(tablearg, None, None) for i in range(self.entries): c[i + 1] = i % 255 # Close the connection. Reopen readonly self.create = False self.close_reopen() c = self.session.open_cursor(tablearg, None, None) i = 0 for key, value in c: self.assertEqual(i + 1, key) self.assertEqual(i % 255, value) i += 1 self.assertEqual(i, self.entries) self.pr('Read %d entries' % i) c.close() self.create = True def test_readonly(self): if self.dirchmod and os.name == 'posix': with self.expectedStderrPattern('Permission'): self.readonly() else: self.readonly()
class test_alter04(wttest.WiredTigerTestCase): name = "alter04" entries = 100 cache_alter = ('1M', '100K') # Settings for os_cache[_dirty]_max. types = [ ('file', dict(uri='file:', use_cg=False, use_index=False)), ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), ('table-index', dict(uri='table:', use_cg=False, use_index=True)), ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), ] sizes = [ ('default', dict(ocreate='')), ('1M', dict(ocreate='1M')), ('200K', dict(ocreate='200K')), ] reopen = [ ('no-reopen', dict(reopen=False)), ('reopen', dict(reopen=True)), ] settings = [ ('cache', dict(setting='os_cache_max')), ('cache_dirty', dict(setting='os_cache_dirty_max')), ] scenarios = make_scenarios(types, sizes, reopen, settings) def verify_metadata(self, metastr): if metastr == '': return cursor = self.session.open_cursor('metadata:', None, None) # # Walk through all the metadata looking for the entries that are # the file URIs for components of the table. # found = False while True: ret = cursor.next() if ret != 0: break key = cursor.get_key() check_meta = ((key.find("lsm:") != -1 or key.find("file:") != -1) \ and key.find(self.name) != -1) if check_meta: value = cursor[key] found = True self.assertTrue(value.find(metastr) != -1) cursor.close() self.assertTrue(found == True) # Alter: Change the setting after creation def test_alter04_cache(self): uri = self.uri + self.name create_params = 'key_format=i,value_format=i,' complex_params = '' # # If we're not explicitly setting the parameter, then don't # modify create_params to test using the default. # if self.ocreate != '': new_param = '%s=%s' % (self.setting, self.ocreate) create_params += '%s,' % new_param complex_params += '%s,' % new_param else: # NOTE: This is hard-coding the default value. If the default # changes then this will fail and need to be fixed. new_param = '%s=0' % self.setting cgparam = '' if self.use_cg or self.use_index: cgparam = 'columns=(k,v),' if self.use_cg: cgparam += 'colgroups=(g0),' self.session.create(uri, create_params + cgparam) # Add in column group or index settings. if self.use_cg: cgparam = 'columns=(v),' suburi = 'colgroup:' + self.name + ':g0' self.session.create(suburi, complex_params + cgparam) if self.use_index: suburi = 'index:' + self.name + ':i0' self.session.create(suburi, complex_params + cgparam) # Put some data in table. c = self.session.open_cursor(uri, None) for k in range(self.entries): c[k + 1] = 1 c.close() # Verify the string in the metadata self.verify_metadata(new_param) # Run through all combinations of the alter commands # for all allowed settings. for a in self.cache_alter: alter_param = '%s=%s' % (self.setting, a) self.session.alter(uri, alter_param) if self.reopen: self.reopen_conn() special = self.use_cg or self.use_index if not special: self.verify_metadata(alter_param) else: self.session.alter(suburi, alter_param) self.verify_metadata(alter_param)
class test_rollback_to_stable14(test_rollback_to_stable_base): session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), ('integer_row', dict(key_format='i')), ] value_format='S' prepare_values = [ ('no_prepare', dict(prepare=False)), ('prepare', dict(prepare=True)) ] scenarios = make_scenarios(key_format_values, prepare_values) def conn_config(self): config = 'cache_size=25MB,statistics=(all),statistics_log=(json,on_close,wait=1),log=(enabled=true),timing_stress_for_test=[history_store_checkpoint_delay]' return config def test_rollback_to_stable(self): nrows = 100 # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" ds = SimpleDataSet( self, uri, 0, key_format=self.key_format, value_format=self.value_format, config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_modQ = mod_val(value_a, 'Q', 0) value_modR = mod_val(value_modQ, 'R', 1) value_modS = mod_val(value_modR, 'S', 2) value_modT = mod_val(value_modS, 'T', 3) value_modW = mod_val(value_modT, 'W', 4) value_modX = mod_val(value_modW, 'X', 5) value_modY = mod_val(value_modX, 'Y', 6) value_modZ = mod_val(value_modY, 'Z', 7) # Perform a combination of modifies and updates. self.pr("large updates and modifies") self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30) self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 40) self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 50) self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) # Verify data is visible and correct. self.check(value_a, uri, nrows, None, 20) self.check(value_modQ, uri, nrows, None, 30) self.check(value_modR, uri, nrows, None, 40) self.check(value_modS, uri, nrows, None, 50) self.check(value_modT, uri, nrows, None, 60) # Pin stable to timestamp 60 if prepare otherwise 50. if self.prepare: self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(60)) else: self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() ckpt = checkpoint_thread(self.conn, done) try: self.pr("start checkpoint") ckpt.start() # Sleep for sometime so that checkpoint starts. time.sleep(2) # Perform several modifies in parallel with checkpoint. # Rollbacks may occur when checkpoint is running, so retry as needed. self.pr("modifies") self.retry_rollback('modify ds1, W', None, lambda: self.large_modifies(uri, 'W', ds, 4, 1, nrows, self.prepare, 70)) self.evict_cursor(uri, nrows, value_modW) self.retry_rollback('modify ds1, X', None, lambda: self.large_modifies(uri, 'X', ds, 5, 1, nrows, self.prepare, 80)) self.evict_cursor(uri, nrows, value_modX) self.retry_rollback('modify ds1, Y', None, lambda: self.large_modifies(uri, 'Y', ds, 6, 1, nrows, self.prepare, 90)) self.evict_cursor(uri, nrows, value_modY) self.retry_rollback('modify ds1, Z', None, lambda: self.large_modifies(uri, 'Z', ds, 7, 1, nrows, self.prepare, 100)) self.evict_cursor(uri, nrows, value_modZ) finally: done.set() ckpt.join() # Simulate a server crash and restart. self.pr("restart") simulate_crash_restart(self, ".", "RESTART") self.pr("restart complete") stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2] hs_restore_updates = stat_cursor[stat.conn.txn_rts_hs_restore_updates][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] stat_cursor.close() self.assertEqual(calls, 0) self.assertEqual(keys_removed, 0) self.assertEqual(hs_restore_updates, nrows) self.assertEqual(keys_restored, 0) if self.prepare: self.assertGreaterEqual(upd_aborted, 0) else: self.assertEqual(upd_aborted, 0) self.assertGreater(pages_visited, 0) self.assertGreaterEqual(hs_removed, nrows) self.assertGreaterEqual(hs_sweep, 0) # Check that the correct data is seen at and after the stable timestamp. self.check(value_a, uri, nrows, None, 20) self.check(value_modQ, uri, nrows, None, 30) self.check(value_modR, uri, nrows, None, 40) self.check(value_modS, uri, nrows, None, 50) # The test may output the following message in eviction under cache pressure. Ignore that. self.ignoreStdoutPatternIfExists("oldest pinned transaction ID rolled back for eviction") def test_rollback_to_stable_same_ts(self): nrows = 100 # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" ds = SimpleDataSet( self, uri, 0, key_format=self.key_format, value_format=self.value_format, config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_modQ = mod_val(value_a, 'Q', 0) value_modR = mod_val(value_modQ, 'R', 1) value_modS = mod_val(value_modR, 'S', 2) value_modT = mod_val(value_modS, 'T', 3) value_modW = mod_val(value_modT, 'W', 4) value_modX = mod_val(value_modW, 'X', 5) value_modY = mod_val(value_modX, 'Y', 6) value_modZ = mod_val(value_modY, 'Z', 7) # Perform a combination of modifies and updates. self.pr("large updates and modifies") self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30) # prepare cannot use same timestamp always, so use a different timestamps that are aborted. if self.prepare: self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 51) self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 55) self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) else: self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 60) self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 60) self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60) # Verify data is visible and correct. self.check(value_a, uri, nrows, None, 20) self.check(value_modQ, uri, nrows, None, 30) self.check(value_modT, uri, nrows, None, 60) self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() ckpt = checkpoint_thread(self.conn, done) try: self.pr("start checkpoint") ckpt.start() # Sleep for sometime so that checkpoint starts. time.sleep(2) # Perform several modifies in parallel with checkpoint. # Rollbacks may occur when checkpoint is running, so retry as needed. self.pr("modifies") self.retry_rollback('modify ds1, W', None, lambda: self.large_modifies(uri, 'W', ds, 4, 1, nrows, self.prepare, 70)) self.evict_cursor(uri, nrows, value_modW) self.retry_rollback('modify ds1, X', None, lambda: self.large_modifies(uri, 'X', ds, 5, 1, nrows, self.prepare, 80)) self.evict_cursor(uri, nrows, value_modX) self.retry_rollback('modify ds1, Y', None, lambda: self.large_modifies(uri, 'Y', ds, 6, 1, nrows, self.prepare, 90)) self.evict_cursor(uri, nrows, value_modY) self.retry_rollback('modify ds1, Z', None, lambda: self.large_modifies(uri, 'Z', ds, 7, 1, nrows, self.prepare, 100)) self.evict_cursor(uri, nrows, value_modZ) finally: done.set() ckpt.join() # Simulate a server crash and restart. self.pr("restart") simulate_crash_restart(self, ".", "RESTART") self.pr("restart complete") stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] hs_restore_updates = stat_cursor[stat.conn.txn_rts_hs_restore_updates][2] hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] stat_cursor.close() self.assertEqual(calls, 0) self.assertEqual(keys_removed, 0) self.assertEqual(hs_restore_updates, nrows) self.assertEqual(keys_restored, 0) if self.prepare: self.assertGreaterEqual(upd_aborted, 0) else: self.assertEqual(upd_aborted, 0) self.assertGreater(pages_visited, 0) self.assertGreaterEqual(hs_removed, nrows * 3) self.assertGreaterEqual(hs_sweep, 0) # Check that the correct data is seen at and after the stable timestamp. self.check(value_a, uri, nrows, None, 20) self.check(value_modQ, uri, nrows, None, 30) # The test may output the following message in eviction under cache pressure. Ignore that. self.ignoreStdoutPatternIfExists("oldest pinned transaction ID rolled back for eviction") def test_rollback_to_stable_same_ts_append(self): nrows = 100 # Create a table without logging. self.pr("create/populate table") uri = "table:rollback_to_stable14" ds = SimpleDataSet( self, uri, 0, key_format=self.key_format, value_format=self.value_format, config='log=(enabled=false)') ds.populate() # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) value_a = "aaaaa" * 100 value_modQ = append_val(value_a, 'Q') value_modR = append_val(value_modQ, 'R') value_modS = append_val(value_modR, 'S') value_modT = append_val(value_modS, 'T') value_modW = append_val(value_modT, 'W') value_modX = append_val(value_modW, 'X') value_modY = append_val(value_modX, 'Y') value_modZ = append_val(value_modY, 'Z') # Perform a combination of modifies and updates. self.pr("large updates and modifies") self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) self.large_modifies(uri, 'Q', ds, len(value_a), 1, nrows, self.prepare, 30) # prepare cannot use same timestamp always, so use a different timestamps that are aborted. if self.prepare: self.large_modifies(uri, 'R', ds, len(value_modQ), 1, nrows, self.prepare, 51) self.large_modifies(uri, 'S', ds, len(value_modR), 1, nrows, self.prepare, 55) self.large_modifies(uri, 'T', ds, len(value_modS), 1, nrows, self.prepare, 60) else: self.large_modifies(uri, 'R', ds, len(value_modQ), 1, nrows, self.prepare, 60) self.large_modifies(uri, 'S', ds, len(value_modR), 1, nrows, self.prepare, 60) self.large_modifies(uri, 'T', ds, len(value_modS), 1, nrows, self.prepare, 60) # Verify data is visible and correct. self.check(value_a, uri, nrows, None, 20) self.check(value_modQ, uri, nrows, None, 30) self.check(value_modT, uri, nrows, None, 60) self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50)) # Create a checkpoint thread done = threading.Event() ckpt = checkpoint_thread(self.conn, done) try: self.pr("start checkpoint") ckpt.start() # Sleep for sometime so that checkpoint starts. time.sleep(2) # Perform several modifies in parallel with checkpoint. # Rollbacks may occur when checkpoint is running, so retry as needed. self.pr("modifies") self.retry_rollback('modify ds1, W', None, lambda: self.large_modifies(uri, 'W', ds, len(value_modT), 1, nrows, self.prepare, 70)) self.retry_rollback('modify ds1, X', None, lambda: self.large_modifies(uri, 'X', ds, len(value_modW), 1, nrows, self.prepare, 80)) self.evict_cursor(uri, nrows, value_modX) self.retry_rollback('modify ds1, Y', None, lambda: self.large_modifies(uri, 'Y', ds, len(value_modX), 1, nrows, self.prepare, 90)) self.retry_rollback('modify ds1, Z', None, lambda: self.large_modifies(uri, 'Z', ds, len(value_modY), 1, nrows, self.prepare, 100)) self.evict_cursor(uri, nrows, value_modZ) finally: done.set() ckpt.join() # Simulate a server crash and restart. self.pr("restart") simulate_crash_restart(self, ".", "RESTART") self.pr("restart complete") stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] hs_restore_updates = stat_cursor[stat.conn.txn_rts_hs_restore_updates][2] hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] stat_cursor.close() self.assertEqual(calls, 0) self.assertEqual(keys_removed, 0) self.assertEqual(hs_restore_updates, nrows) self.assertEqual(keys_restored, 0) if self.prepare: self.assertGreaterEqual(upd_aborted, 0) else: self.assertEqual(upd_aborted, 0) self.assertGreater(pages_visited, 0) self.assertGreaterEqual(hs_removed, nrows * 3) self.assertGreaterEqual(hs_sweep, 0) # Check that the correct data is seen at and after the stable timestamp. self.check(value_a, uri, nrows, None, 20) self.check(value_modQ, uri, nrows, None, 30) # The test may output the following message in eviction under cache pressure. Ignore that. self.ignoreStdoutPatternIfExists("oldest pinned transaction ID rolled back for eviction")
class test_rollback_to_stable06(test_rollback_to_stable_base): format_values = [ ('column', dict(key_format='r', value_format='S')), ('column_fix', dict(key_format='r', value_format='8t')), ('row_integer', dict(key_format='i', value_format='S')), ] in_memory_values = [('no_inmem', dict(in_memory=False)), ('inmem', dict(in_memory=True))] prepare_values = [('no_prepare', dict(prepare=False)), ('prepare', dict(prepare=True))] scenarios = make_scenarios(format_values, in_memory_values, prepare_values) def conn_config(self): config = 'cache_size=50MB,statistics=(all)' if self.in_memory: config += ',in_memory=true' else: config += ',log=(enabled),in_memory=false' return config def test_rollback_to_stable(self): nrows = 1000 # Create a table without logging. uri = "table:rollback_to_stable06" ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format, config='log=(enabled=false)') ds.populate() if self.value_format == '8t': value_a = 97 value_b = 98 value_c = 99 value_d = 100 else: value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 value_c = "ccccc" * 100 value_d = "ddddd" * 100 # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) # Perform several updates. self.large_updates(uri, value_a, ds, nrows, self.prepare, 20) self.large_updates(uri, value_b, ds, nrows, self.prepare, 30) self.large_updates(uri, value_c, ds, nrows, self.prepare, 40) self.large_updates(uri, value_d, ds, nrows, self.prepare, 50) # Verify data is visible and correct. self.check(value_a, uri, nrows, None, 20) self.check(value_b, uri, nrows, None, 30) self.check(value_c, uri, nrows, None, 40) self.check(value_d, uri, nrows, None, 50) # Checkpoint to ensure the data is flushed, then rollback to the stable timestamp. if not self.in_memory: self.session.checkpoint() self.conn.rollback_to_stable() # Check that all keys are removed. # (For FLCS, at least for now, they will read back as 0, meaning deleted, rather # than disappear.) self.check(value_a, uri, 0, nrows, 20) self.check(value_b, uri, 0, nrows, 30) self.check(value_c, uri, 0, nrows, 40) self.check(value_d, uri, 0, nrows, 50) stat_cursor = self.session.open_cursor('statistics:', None, None) calls = stat_cursor[stat.conn.txn_rts][2] hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2] pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2] upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2] stat_cursor.close() self.assertEqual(calls, 1) self.assertEqual(keys_restored, 0) self.assertGreater(pages_visited, 0) self.assertGreaterEqual(keys_removed, 0) if self.in_memory: self.assertEqual(upd_aborted, nrows * 4) self.assertEqual(hs_removed, 0) else: self.assertGreaterEqual(upd_aborted + hs_removed + keys_removed, nrows * 4)
class test_compact02(wttest.WiredTigerTestCase): types = [ ('table', dict(uri='table:test_compact02')), ] cacheSize = [ ('default', dict(cacheSize='')), ('1mb', dict(cacheSize='cache_size=1MB')), ('10gb', dict(cacheSize='cache_size=10GB')), ] # There's a balance between the pages we create and the size of the records # being stored: compaction doesn't work on tables with many overflow items # because we don't rewrite them. Experimentally, 8KB is as small as the test # can go. Additionally, we can't set the maximum page size too large because # there won't be enough pages to rewrite. Experimentally, 128KB works. fileConfig = [ ('default', dict(fileConfig='')), ('8KB', dict(fileConfig='leaf_page_max=8kb')), ('64KB', dict(fileConfig='leaf_page_max=64KB')), ('128KB', dict(fileConfig='leaf_page_max=128KB')), ] scenarios = make_scenarios(types, cacheSize, fileConfig) # We want about 22K records that total about 130Mb. That is an average # of 6196 bytes per record. Half the records should be smaller, about # 2700 bytes (about 30Mb) and the other half should be larger, 9666 bytes # per record (about 100Mb). # # Test flow is as follows. # # 1. Create a table with the data, alternating record size. # 2. Checkpoint and get stats on the table to confirm the size. # 3. Delete the half of the records with the larger record size. # 4. Checkpoint so compact finds something to work with. # 5. Call compact. # 6. Get stats on compacted table. # nrecords = 22000 bigvalue = "abcdefghi" * 1074 # 9*1074 == 9666 smallvalue = "ihgfedcba" * 303 # 9*303 == 2727 fullsize = nrecords / 2 * len(bigvalue) + nrecords / 2 * len(smallvalue) # Return the size of the file def getSize(self): # To allow this to work on systems without ftruncate, # get the portion of the file allocated, via 'statistics=(all)', # not the physical file size, via 'statistics=(size)'. cstat = self.session.open_cursor('statistics:' + self.uri, None, 'statistics=(all)') sz = cstat[stat.dsrc.block_size][2] cstat.close() return sz # This test varies the cache size and so needs to set up its own connection. # Override the standard methods. def setUpConnectionOpen(self, dir): return None def setUpSessionOpen(self, conn): return None def ConnectionOpen(self, cacheSize): self.home = '.' conn_params = 'create,' + \ cacheSize + ',error_prefix="%s: ",' % self.shortid() + \ 'statistics=(all),' + \ 'eviction_dirty_target=99,eviction_dirty_trigger=99' try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: print "Failed conn at '%s' with config '%s'" % (dir, conn_params) self.session = self.conn.open_session(None) # Create a table, add keys with both big and small values. def test_compact02(self): self.ConnectionOpen(self.cacheSize) mb = 1024 * 1024 params = 'key_format=i,value_format=S,' + self.fileConfig # 1. Create a table with the data, alternating record size. self.session.create(self.uri, params) c = self.session.open_cursor(self.uri, None) for i in range(self.nrecords): if i % 2 == 0: c[i] = str(i) + self.bigvalue else: c[i] = str(i) + self.smallvalue c.close() # 2. Checkpoint and get stats on the table to confirm the size. self.session.checkpoint() sz = self.getSize() self.pr('After populate ' + str(sz / mb) + 'MB') self.assertGreater(sz, self.fullsize) # 3. Delete the half of the records with the larger record size. c = self.session.open_cursor(self.uri, None) count = 0 for i in range(self.nrecords): if i % 2 == 0: count += 1 c.set_key(i) c.remove() c.close() self.pr('Removed total ' + str((count * 9666) / mb) + 'MB') # 4. Checkpoint self.session.checkpoint() # 5. Call compact. self.session.compact(self.uri, None) # 6. Get stats on compacted table. sz = self.getSize() self.pr('After compact ' + str(sz / mb) + 'MB') # After compact, the file size should be less than half the full size. self.assertLess(sz, self.fullsize / 2)
class test_prepare_lookaside02(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_prepare_cursor' uri = 'table:' + tablename txn_config = 'isolation=snapshot' types = [ ('col', dict(s_config='value_format=i,log=(enabled=false),key_format=r')), ('row', dict(s_config='key_format=i,value_format=i,log=(enabled=false)')), ('lsm', dict(s_config= 'key_format=i, value_format=i,log=(enabled=false),type=lsm')), ] # Transaction end types txn_end = [ ('txn_commit', dict(txn_commit=True)), ('txn_rollback', dict(txn_commit=False)), ] scenarios = make_scenarios(types, txn_end) def test_prepare_conflict(self): self.session.create(self.uri, self.s_config) c = self.session.open_cursor(self.uri) # Insert keys 1..100 each with timestamp=key, in some order orig_keys = range(1, 101) keys = orig_keys[:] random.shuffle(keys) # Scenario: 1 # Check insert operation self.session.begin_transaction(self.txn_config) c[1] = 1 # update the value with in this transaction self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(100)) if self.txn_commit == True: self.session.commit_transaction('commit_timestamp=' + timestamp_str(101) + ',durable_timestamp=' + timestamp_str(101)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation self.conn.set_timestamp('stable_timestamp=' + timestamp_str(150)) self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(150)) self.session.checkpoint() # Scenario: 2 # Check update operation # update a existing key. # update a newly inserted key with in this transaction self.session.begin_transaction(self.txn_config) # update a committed value, key 1 is inserted above. c[1] = 2 # update a uncommitted value, insert and update a key. c[2] = 1 c[2] = 2 self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) if self.txn_commit == True: self.session.commit_transaction('commit_timestamp=' + timestamp_str(201) + ',durable_timestamp=' + timestamp_str(201)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) self.session.checkpoint() # Scenario: 3 # Check remove operation # remove an existing key. # remove a previously updated key. # remove a newly inserted and updated key. self.session.begin_transaction(self.txn_config) # update a committed value, key 1 is inserted above. c.set_key(1) c.remove() c.set_key(2) c.remove() c[3] = 1 c[3] = 2 c.set_key(3) c.remove() self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(300)) if self.txn_commit == True: self.session.commit_transaction('commit_timestamp=' + timestamp_str(301) + ',durable_timestamp=' + timestamp_str(301)) else: self.session.rollback_transaction() # commit some keys, to generate the update chain subsequently. self.session.begin_transaction(self.txn_config) c[1] = 1 c[2] = 1 c[3] = 1 self.session.commit_transaction('commit_timestamp=' + timestamp_str(301)) # Trigger a checkpoint, which could trigger reconciliation self.conn.set_timestamp('stable_timestamp=' + timestamp_str(350)) self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(350)) self.session.checkpoint() #Scenario: 4 # Check update operation on a checkpointed key. Re-open is to facilitate # creating the modify update_chain for key instead of insert update # chain. self.reopen_conn() self.conn.set_timestamp('stable_timestamp=' + timestamp_str(350)) self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(350)) self.session.create(self.uri, self.s_config) cur = self.session.open_cursor(self.uri) self.session.begin_transaction(self.txn_config) cur[1] = 2 cur[2] = 2 cur[3] = 2 # Update a key twice cur[2] = 3 # Remove a updated key cur.set_key(3) cur.remove() self.session.prepare_transaction('prepare_timestamp=' + timestamp_str(400)) if self.txn_commit == True: self.session.commit_transaction('commit_timestamp=' + timestamp_str(401) + ',durable_timestamp=' + timestamp_str(401)) else: self.session.rollback_transaction() # Trigger a checkpoint, which could trigger reconciliation self.conn.set_timestamp('stable_timestamp=' + timestamp_str(450)) self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(450)) self.session.checkpoint() cur.close() self.session.close()
class test_durable_rollback_to_stable(wttest.WiredTigerTestCase, suite_subprocess): session_config = 'isolation=snapshot' keyfmt = [ ('row-string', dict(keyfmt='S')), ('row-int', dict(keyfmt='i')), # The commented columnar tests needs to be enabled once rollback to stable for columnar is fixed in (WT-5548). # ('column-store', dict(keyfmt='r')), ] types = [ ('file', dict(uri='file', ds=SimpleDataSet)), ('lsm', dict(uri='lsm', ds=SimpleDataSet)), ('table-simple', dict(uri='table', ds=SimpleDataSet)), ] iso_types = [('isolation_read_committed', dict(isolation='read-committed')), ('isolation_default', dict(isolation='')), ('isolation_snapshot', dict(isolation='snapshot'))] scenarios = make_scenarios(types, keyfmt, iso_types) def skip(self): return self.keyfmt == 'r' and \ (self.ds.is_lsm() or self.uri == 'lsm') # Test durable timestamp. def test_durable_rollback_to_stable(self): if self.skip(): return # Build an object. uri = self.uri + ':test_durable_rollback_to_stable' ds = self.ds(self, uri, 50, key_format=self.keyfmt) ds.populate() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) # Set stable timestamp to checkpoint initial data set. self.conn.set_timestamp('stable_timestamp=' + timestamp_str(100)) self.session.checkpoint() # Update all values with value 111 i.e. first update value. session.begin_transaction() self.assertEquals(cursor.next(), 0) for i in range(1, 50): cursor.set_value(ds.value(111)) self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) session.prepare_transaction('prepare_timestamp=' + timestamp_str(150)) session.timestamp_transaction('commit_timestamp=' + timestamp_str(200)) session.timestamp_transaction('durable_timestamp=' + timestamp_str(220)) session.commit_transaction() # Check the values read are correct with different timestamps. # Read the initial dataset. self.assertEquals(cursor.reset(), 0) session.begin_transaction('read_timestamp=' + timestamp_str(150)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(i)) self.assertEquals(cursor.next(), 0) session.commit_transaction() # Read the first update value with timestamp. self.assertEquals(cursor.reset(), 0) session.begin_transaction('read_timestamp=' + timestamp_str(200)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) self.assertEquals(cursor.next(), 0) session.commit_transaction() # Check that latest value is same as first update value. self.assertEquals(cursor.reset(), 0) session.begin_transaction() self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) self.assertEquals(cursor.next(), 0) session.commit_transaction() # Set a stable timestamp so that first update value is durable. self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) # Update all values with value 222 i.e. second update value. self.assertEquals(cursor.reset(), 0) session.begin_transaction() self.assertEquals(cursor.next(), 0) for i in range(1, 50): cursor.set_value(ds.value(222)) self.assertEquals(cursor.update(), 0) self.assertEquals(cursor.next(), 0) session.prepare_transaction('prepare_timestamp=' + timestamp_str(200)) # Commit timestamp is earlier to stable timestamp but durable timestamp # is later than stable timestamp. Hence second update value is not durable. session.timestamp_transaction('commit_timestamp=' + timestamp_str(240)) session.timestamp_transaction('durable_timestamp=' + timestamp_str(300)) session.commit_transaction() # Checkpoint so that first update value will be visible and durable, # but second update value will be only visible but not durable. self.session.checkpoint() # Check that second update value is visible. self.assertEquals(cursor.reset(), 0) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(222)) self.assertEquals(cursor.next(), 0) cursor.close() session.close() # Check that second update value was not durable by reopening. self.conn.rollback_to_stable() session = self.conn.open_session(self.session_config) cursor = session.open_cursor(uri, None) self.conn.set_timestamp('stable_timestamp=' + timestamp_str(250)) self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(250)) self.assertEquals(cursor.next(), 0) for i in range(1, 50): self.assertEquals(cursor.get_value(), ds.value(111)) self.assertEquals(cursor.next(), 0) # Use util to verify that second updates values have been flushed. errfilename = "verifyrollbackerr.out" self.runWt(["verify", "-s", uri], errfilename=errfilename, failure=False) self.check_empty_file(errfilename)
class test_compat04(wttest.WiredTigerTestCase, suite_subprocess): # Add enough entries and use a small log size to generate more than # one log file. entries = 2000 logmax = "100K" tablename = 'test_compat04' uri = 'table:' + tablename # Declare the log versions that do and do not have prevlsn. # Log version 1 does not have the prevlsn record. # Log version 2 introduced that record. # Log versions 3 and higher continue to have that record. min_logv = 2 # The outline of this test is that we create the database at the # create release value. Then we reconfigure the release to the # reconfig release value. Then we close and reopen the database with # a release and compatibility maximum of that release value. This # should be successful for all directions. # create_release = [ ('def_rel', dict(create_rel='none', log_crrel=5)), ('100_rel', dict(create_rel="10.0", log_crrel=5)), ('33_rel', dict(create_rel="3.3", log_crrel=4)), ('32_rel', dict(create_rel="3.2", log_crrel=3)), ('31_rel', dict(create_rel="3.1", log_crrel=3)), ('30_rel', dict(create_rel="3.0", log_crrel=2)), ('26_rel', dict(create_rel="2.6", log_crrel=1)), ] reconfig_release = [ ('100_rel', dict(rel="10.0", log_rel=5)), ('33_rel', dict(rel="3.3", log_rel=4)), ('32_rel', dict(rel="3.2", log_rel=3)), ('31_rel', dict(rel="3.1", log_rel=3)), ('30_rel', dict(rel="3.0", log_rel=2)), ('300_rel', dict(rel="3.0.0", log_rel=2)), ('26_rel', dict(rel="2.6", log_rel=1)), ] base_config = [ ('basecfg_true', dict(basecfg='true')), ('basecfg_false', dict(basecfg='false')), ] scenarios = make_scenarios(create_release, reconfig_release, base_config) # This test creates scenarios that lead to errors. This is different # than compat02 because it is testing errors (or success) using the # compatibility settings on the initial database creation. def conn_config(self): config_str = 'create,config_base=%s,' % self.basecfg log_str = 'log=(enabled,file_max=%s,remove=false),' % self.logmax compat_str = '' if (self.create_rel != 'none'): compat_str += 'compatibility=(release="%s"),' % self.create_rel config_str += log_str + compat_str self.pr("Conn config:" + config_str) return config_str def test_compat04(self): # # Create initial database at the compatibility level requested # and a table with some data. # self.session.create(self.uri, 'key_format=i,value_format=i') c = self.session.open_cursor(self.uri, None) # # Add some entries to generate log files. # for i in range(self.entries): c[i] = i + 1 c.close() # Reconfigure and close the connection. Then reopen with that release. # We expect success. config_str = 'compatibility=(release=%s)' % self.rel self.conn.reconfigure(config_str) self.conn.close() config_str = 'compatibility=(release=%s,require_max=%s)' % (self.rel, self.rel) conn = self.wiredtiger_open('.', config_str) conn.close()
class test_prepare10(wttest.WiredTigerTestCase): # Force a small cache. conn_config = 'cache_size=10MB,eviction_dirty_trigger=80,eviction_updates_trigger=80' session_config = 'isolation=snapshot' key_format_values = [ ('column', dict(key_format='r')), ('string-row', dict(key_format='S')), ] scenarios = make_scenarios(key_format_values) def updates(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) self.session.begin_transaction() for i in range(1, nrows): cursor.set_key(ds.key(i)) cursor.set_value(value) self.assertEquals(cursor.insert(), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def removes(self, ds, uri, nrows, ts): cursor = self.session.open_cursor(uri) self.session.begin_transaction() for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.remove(), 0) self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(ts)) cursor.close() def check(self, ds, uri, nrows, value, ts): cursor = self.session.open_cursor(uri) self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.search(), 0) self.assertEquals(cursor.get_value(),value) self.session.commit_transaction() cursor.close() def check_not_found(self, ds, uri, nrows, ts): cursor = self.session.open_cursor(uri) self.session.begin_transaction('ignore_prepare=true,read_timestamp=' + self.timestamp_str(ts)) for i in range(1, nrows): cursor.set_key(ds.key(i)) self.assertEquals(cursor.search(), wiredtiger.WT_NOTFOUND) self.session.commit_transaction() cursor.close() def test_prepare_rollback_retrieve_time_window(self): # Create a small table. uri = "table:test_prepare10" nrows = 1000 ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format='u') ds.populate() value_a = b"aaaaa" * 100 value_b = b"bbbbb" * 100 value_c = b"ccccc" * 100 # Commit some updates along with a prepared update, which is not resolved. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10)) self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10)) # Initially load huge data self.updates(ds, uri, nrows, value_a, 20) # Add some more updates self.updates(ds, uri, nrows, value_b, 30) # Checkpoint self.session.checkpoint() # Validate that we do see the correct value. session2 = self.setUpSessionOpen(self.conn) cursor2 = session2.open_cursor(uri) session2.begin_transaction() for i in range(1, nrows): cursor2.set_key(ds.key(i)) self.assertEquals(cursor2.search(), 0) self.assertEquals(cursor2.get_value(), value_b) session2.commit_transaction() # Reset the cursor. cursor2.reset() session2.begin_transaction() # Remove all keys self.removes(ds, uri, nrows, 40) # Validate that we do see the correct value. session3 = self.setUpSessionOpen(self.conn) cursor3 = session3.open_cursor(uri) session3.begin_transaction() for i in range(1, nrows): cursor3.set_key(ds.key(i)) self.assertEquals(cursor3.search(), wiredtiger.WT_NOTFOUND) session3.commit_transaction() # Reset the cursor. cursor3.reset() session3.begin_transaction() # Insert the updates from a prepare session and keep it open. session_p = self.conn.open_session() cursor_p = session_p.open_cursor(uri) session_p.begin_transaction() for i in range(1, nrows): cursor_p.set_key(ds.key(i)) cursor_p.set_value(value_c) self.assertEquals(cursor_p.insert(), 0) session_p.prepare_transaction('prepare_timestamp=' + self.timestamp_str(50)) self.check(ds, uri, nrows, value_a, 20) self.check(ds, uri, nrows, value_b, 35) self.check_not_found(ds, uri, nrows, 60) #rollback the prepared session session_p.rollback_transaction() self.check(ds, uri, nrows, value_a, 20) self.check(ds, uri, nrows, value_b, 35) self.check_not_found(ds, uri, nrows, 60) # session2 still can see the value_b for i in range(1, nrows): cursor2.set_key(ds.key(i)) self.assertEquals(cursor2.search(), 0) self.assertEquals(cursor2.get_value(), value_b) session2.commit_transaction() # session3 still can't see a value for i in range(1, nrows): cursor3.set_key(ds.key(i)) self.assertEquals(cursor3.search(), wiredtiger.WT_NOTFOUND) session3.commit_transaction() # close sessions. cursor_p.close() session_p.close() cursor2.close() session2.close() cursor3.close() session3.close() self.session.close()
class test_schema04(wttest.WiredTigerTestCase): """ Test indices with duplicates. Our set of rows looks like a multiplication table: row 0: [ 0, 0, 0, 0, 0, 0 ] row 1: [ 0, 1, 2, 3, 4, 5 ] row 2: [ 0, 2, 4, 6, 8, 10 ] with the twist that entries are mod 100. So, looking further: row 31: [ 0, 31, 62, 93, 24, 55 ] Each column is placed into its own index. The mod twist, as well as the 0th column, guarantees we'll have some duplicates. """ nentries = 100 scenarios = make_scenarios([ ('index-before', { 'create_index' : 0 }), ('index-during', { 'create_index' : 1 }), ('index-after', { 'create_index' : 2 }), ]) def create_indices(self): # Create 6 index files, each with a column from the main table for i in range(0, 6): self.session.create("index:schema04:x" + str(i), "key_format=i,columns=(v" + str(i) + "),") # We split the population into two phases # (in anticipation of future tests that create # indices between the two population steps). def populate(self, phase): cursor = self.session.open_cursor('table:schema04', None, None) if phase == 0: range_from = 0 range_to = self.nentries // 2 else: range_from = self.nentries // 2 range_to = self.nentries for i in range(range_from, range_to): # e.g. element 31 is '0,31,62,93,24,55' cursor.set_key(i) cursor.set_value( (i*0)%100, (i*1)%100, (i*2)%100, (i*3)%100, (i*4)%100, (i*5)%100) cursor.insert() cursor.close() def check_entries(self): cursor = self.session.open_cursor('table:schema04', None, None) icursor = [] for i in range(0, 6): icursor.append(self.session.open_cursor('index:schema04:x' + str(i), None, None)) i = 0 for kv in cursor: # Check main table expect = [(i*j)%100 for j in range(0, 6)] primkey = kv.pop(0) self.assertEqual(i, primkey) self.assertEqual(kv, expect) for j in range(0, 6): self.assertEqual((i*j)%100, kv[j]) for idx in range(0, 6): c = icursor[idx] indexkey = (i*idx)%100 c.set_key(indexkey) self.assertEqual(c.search(), 0) value = c.get_value() while value != expect and value[idx] == expect[idx]: c.next() value = c.get_value() self.assertEqual(value, expect) i += 1 self.assertEqual(self.nentries, i) def test_index(self): self.session.create("table:schema04", "key_format=i,value_format=iiiiii," "columns=(primarykey,v0,v1,v2,v3,v4,v5)") if self.create_index == 0: self.create_indices() self.populate(0) if self.create_index == 1: self.create_indices() self.populate(1) if self.create_index == 2: self.create_indices() self.check_entries()
class test_dump(wttest.WiredTigerTestCase, suite_subprocess): dir = 'dump.dir' # Backup directory name name = 'test_dump' name2 = 'test_dumpb' nentries = 2500 dumpfmt = [('hex', dict(hex=1)), ('txt', dict(hex=0))] keyfmt = [('integer', dict(keyfmt='i')), ('recno', dict(keyfmt='r')), ('string', dict(keyfmt='S'))] types = [ ('file', dict(uri='file:', dataset=SimpleDataSet)), ('lsm', dict(uri='lsm:', dataset=SimpleDataSet)), ('table-simple', dict(uri='table:', dataset=SimpleDataSet)), ('table-index', dict(uri='table:', dataset=SimpleIndexDataSet)), ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)), ('table-complex', dict(uri='table:', dataset=ComplexDataSet)), ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)), ('table-simple-proj', dict(uri='table:', dataset=ProjectionDataSet, projection=True)), ('table-index-proj', dict(uri='table:', dataset=ProjectionIndexDataSet, projection=True)), ] scenarios = make_scenarios(types, keyfmt, dumpfmt) def skip(self): return (self.dataset.is_lsm() or self.uri == 'lsm:') and \ self.keyfmt == 'r' # Extract the values lines from the dump output. def value_lines(self, fname): # mode: # 0 == we are in the header # 1 == next line is key # 2 == next line is value mode = 0 lines = [] for line in open(fname).readlines(): if mode == 0: if line == 'Data\n': mode = 1 elif mode == 1: mode = 2 else: # This is a value line, keep it. lines.append(line) mode = 1 return sorted(lines) def compare_dump_values(self, f1, f2): l1 = self.value_lines(f1) l2 = self.value_lines(f2) self.assertEqual(l1, l2) # Dump, re-load and do a content comparison. def test_dump(self): # LSM and column-store isn't a valid combination. if self.skip(): return # Create the object. uri = self.uri + self.name uri2 = self.uri + self.name2 pop = self.dataset(self, uri, self.nentries, key_format=self.keyfmt) pop.populate() # Dump the object. os.mkdir(self.dir) if self.hex == 1: self.runWt(['dump', '-x', uri], outfilename='dump.out') else: self.runWt(['dump', uri], outfilename='dump.out') # Re-load the object. self.runWt(['-h', self.dir, 'load', '-f', 'dump.out']) # Check the database contents self.runWt(['list'], outfilename='list.out') self.runWt(['-h', self.dir, 'list'], outfilename='list.out.new') s1 = set(open('list.out').read().split()) s2 = set(open('list.out.new').read().split()) self.assertEqual(not s1.symmetric_difference(s2), True) # Check the object's contents self.reopen_conn(self.dir) pop.check() # Re-load the object again in the original directory. self.reopen_conn('.') self.runWt(['-h', self.dir, 'load', '-f', 'dump.out']) # Check the contents, they shouldn't have changed. pop.check() # Re-load the object again, but confirm -n (no overwrite) fails. self.runWt(['-h', self.dir, 'load', '-n', '-f', 'dump.out'], errfilename='errfile.out', failure=True) self.check_non_empty_file('errfile.out') # If there are indices, dump one of them and check the output. if self.dataset == ComplexDataSet: indexuri = 'index:' + self.name + ':indx1' hexopt = ['-x'] if self.hex == 1 else [] self.runWt(['-h', self.dir, 'dump'] + hexopt + [indexuri], outfilename='dumpidx.out') self.check_non_empty_file('dumpidx.out') self.compare_dump_values('dump.out', 'dumpidx.out') # Re-load the object into a different table uri shutil.rmtree(self.dir) os.mkdir(self.dir) self.runWt( ['-h', self.dir, 'load', '-r', self.name2, '-f', 'dump.out']) # Check the contents in the new table. self.reopen_conn(self.dir) pop = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt) pop.check()
class test_cursor_random_invisible(wttest.WiredTigerTestCase): types = [('file', dict(type='file:random')), ('table', dict(type='table:random'))] config = [('sample', dict(config='next_random=true,next_random_sample_size=35')), ('not-sample', dict(config='next_random=true'))] scenarios = make_scenarios(types, config) def test_cursor_random_invisible_all(self): uri = self.type self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None) # Start a transaction. self.session.begin_transaction() for i in range(1, 100): cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Open another session, the updates won't yet be visible, we shouldn't # find anything at all. s = self.conn.open_session() cursor = s.open_cursor(uri, None, self.config) self.assertEqual(cursor.next(), wiredtiger.WT_NOTFOUND) def test_cursor_random_invisible_after(self): uri = self.type self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None) # Insert a single leading record. cursor[simple_key(cursor, 1)] = simple_value(cursor, 1) # Start a transaction. self.session.begin_transaction() for i in range(2, 100): cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Open another session, the updates won't yet be visible, we should # return the only possible record. s = self.conn.open_session() cursor = s.open_cursor(uri, None, self.config) self.assertEquals(cursor.next(), 0) self.assertEqual(cursor.get_key(), simple_key(cursor, 1)) def test_cursor_random_invisible_before(self): uri = self.type self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None) # Insert a single leading record. cursor[simple_key(cursor, 99)] = simple_value(cursor, 99) # Start a transaction. self.session.begin_transaction() for i in range(2, 100): cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Open another session, the updates won't yet be visible, we should # return the only possible record. s = self.conn.open_session() cursor = s.open_cursor(uri, None, self.config) self.assertEquals(cursor.next(), 0) self.assertEqual(cursor.get_key(), simple_key(cursor, 99))
class test_compat01(wttest.WiredTigerTestCase, suite_subprocess): # Add enough entries and use a small log size to generate more than # one log file. entries = 2000 logmax = "100K" tablename = 'test_compat01' uri = 'table:' + tablename sync_list = [ '(method=fsync,enabled)', '(method=none,enabled)', ] # The API uses only the major and minor numbers but accepts with # and without the patch number. Test both. start_compat = [ ('def', dict(compat1='none', current1=True)), ('current', dict(compat1="3.0", current1=True)), ('current_patch', dict(compat1="3.0.0", current1=True)), ('minor_only', dict(compat1="2.6", current1=False)), ('minor_patch', dict(compat1="2.6.1", current1=False)), ('old', dict(compat1="1.8", current1=False)), ('old_patch', dict(compat1="1.8.1", current1=False)), ] restart_compat = [ ('def2', dict(compat2='none', current2=True)), ('current2', dict(compat2="3.0", current2=True)), ('current_patch2', dict(compat2="3.0.0", current2=True)), ('minor_only2', dict(compat2="2.6", current2=False)), ('minor_patch2', dict(compat2="2.6.1", current2=False)), ('old2', dict(compat2="1.8", current2=False)), ('old_patch2', dict(compat2="1.8.1", current2=False)), ] scenarios = make_scenarios(restart_compat, start_compat) def make_compat_str(self, create): compat_str = '' if (create == True and self.compat1 != 'none'): #compat_str = 'verbose=(temporary),compatibility=(release="%s"),' % self.compat1 compat_str = 'compatibility=(release="%s"),' % self.compat1 elif (create == False and self.compat2 != 'none'): #compat_str = 'verbose=(temporary),compatibility=(release="%s"),' % self.compat2 compat_str = 'compatibility=(release="%s"),' % self.compat2 return compat_str def conn_config(self): # Cycle through the different transaction_sync values in a # deterministic manner. txn_sync = self.sync_list[self.scenario_number % len(self.sync_list)] # Set archive false on the home directory. log_str = 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ 'transaction_sync="%s",' % txn_sync compat_str = self.make_compat_str(True) self.pr("Conn config:" + log_str + compat_str) return log_str + compat_str def check_prev_lsn(self, conn_close, prev_lsn_count): # # Run printlog and look for the prev_lsn log record. Verify its # existence with the passed in expected result. We don't use # check_file_contains because that only looks in the first 100K and # we don't know how big our text-based log output is. Look through # the entire file if needed and set a boolean for comparison. # self.runWt(['printlog'], outfilename='printlog.out', closeconn=conn_close) contains = 0 with open('printlog.out') as logfile: for line in logfile: if 'prev_lsn' in line: contains += 1 self.assertEqual(prev_lsn_count, contains) def check_log(self, reconfig): orig_logs = fnmatch.filter(os.listdir('.'), "*gerLog*") compat_str = self.make_compat_str(False) if self.current1: prev_lsn_logs = len(orig_logs) else: prev_lsn_logs = 0 if not reconfig: # # Close and open the connection to force recovery and reset the # compatibility string on startup. # self.conn.close() log_str = 'log=(enabled,file_max=%s,archive=false),' % self.logmax restart_config = log_str + compat_str self.pr("Restart conn " + restart_config) # # Open a connection to force it to run recovery. # conn = self.wiredtiger_open('.', restart_config) conn.close() check_close = False # # If the version was upgraded we'll see a new log file containing # the new log record no matter what the original setting was. # if self.current2: prev_lsn_logs += 1 else: self.pr("Reconfigure: " + compat_str) self.conn.reconfigure(compat_str) check_close = True # # If we're reconfiguring, we'll only see another new log file # when upgrading. Staying at the same version has no effect. # if self.current2 and not self.current1: prev_lsn_logs += 1 # Run printlog and verify the new record does or does not exist. # Need to check count of log files that should and should not have # the prev_lsn record based on the count of log files that exist # before and after. Pass that into this function and check the # number of prev_lsn records we see. self.check_prev_lsn(check_close, prev_lsn_logs) def run_test(self, reconfig): # If reconfiguring with the empty string there is nothing to do. if reconfig == True and self.compat2 == 'none': return self.session.create(self.uri, 'key_format=i,value_format=i') c = self.session.open_cursor(self.uri, None) # # Add some entries to generate log files. # for i in range(self.entries): c[i] = i + 1 c.close() # Check the log state after the entire op completes # and run recovery with the restart compatibility mode. self.check_log(reconfig) # Run the same test but reset the compatibility via # reconfigure or changing it when reopening the connection. def test_reconfig(self): self.run_test(True) def test_restart(self): self.run_test(False)
class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): logmax = "100K" tablename = 'test_txn07' uri = 'table:' + tablename remove_list = ['true', 'false'] sync_list = [ '(method=dsync,enabled)', '(method=fsync,enabled)', '(method=none,enabled)', '(enabled=false)' ] types = [ ('row', dict(tabletype='row', create_params = 'key_format=i,value_format=S')), ('var', dict(tabletype='var', create_params = 'key_format=r,value_format=S')), ('fix', dict(tabletype='fix', create_params = 'key_format=r,value_format=8t')), ] op1s = [ ('trunc-all', dict(op1=('all', 0))), ('trunc-both', dict(op1=('both', 2))), ('trunc-start', dict(op1=('start', 2))), ('trunc-stop', dict(op1=('stop', 2))), ] txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))] compress = [ ('nop', dict(compress='nop')), ('snappy', dict(compress='snappy')), ('zlib', dict(compress='zlib')), ('none', dict(compress='')), ] scenarios = make_scenarios(types, op1s, txn1s, compress, prune=30, prunelong=1000) def conn_config(self): return 'log=(enabled,file_max=%s,' % self.logmax + \ 'compressor=%s,remove=false)' % self.compress + \ ',create,error_prefix="%s",' % self.shortid() + \ "statistics=(fast)," + \ 'transaction_sync="%s",' % \ self.sync_list[self.scenario_number % len(self.sync_list)] def conn_extensions(self, extlist): extlist.skip_if_missing = True extlist.extension('compressors', self.compress) # Check that a cursor (optionally started in a new transaction), sees the # expected values. def check(self, session, txn_config, expected): if txn_config: session.begin_transaction(txn_config) c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: session.commit_transaction() self.assertEqual(actual, expected) # Check the state of the system with respect to the current cursor and # different isolation levels. def check_all(self, current, committed): # Transactions see their own changes. # Read-uncommitted transactions see all changes. # Snapshot and read-committed transactions should not see changes. self.check(self.session, None, current) self.check(self.session2, "isolation=snapshot", committed) self.check(self.session2, "isolation=read-committed", committed) self.check(self.session2, "isolation=read-uncommitted", current) # Opening a clone of the database home directory should run # recovery and see the committed results. Flush the log because # the backup may not get all the log records if we are running # without a sync option. Use sync=off to force a write to the OS. self.session.log_flush('sync=off') self.backup(self.backup_dir) backup_conn_params = 'log=(enabled,file_max=%s,' % self.logmax + \ 'compressor=%s)' % self.compress + \ self.extensionsConfig() backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: self.check(backup_conn.open_session(), None, committed) finally: backup_conn.close() def test_ops(self): self.backup_dir = os.path.join(self.home, "WT_BACKUP") self.session2 = self.conn.open_session() # print "Creating %s with config '%s'" % (self.uri, self.create_params) self.session.create(self.uri, self.create_params) # Set up the table with entries for 1-5. # We then truncate starting or ending in various places. c = self.session.open_cursor(self.uri, None) if self.tabletype == 'fix': value = 1 else: # Choose large compressible values for the string cases. value = 'abc' * 1000000 current = {1:value, 2:value, 3:value, 4:value, 5:value} for k in current: c[k] = value committed = current.copy() ops = (self.op1, ) txns = (self.txn1, ) for i, ot in enumerate(zip(ops, txns)): self.session.begin_transaction() ok, txn = ot # print '%d: %s(%d)[%s]' % (i, ok[0], ok[1], txn) op, k = ok # print '%d: %s(%d)[%s]' % (i, ok[0], ok[1], txn) if op == 'stop': c.set_key(k) self.session.truncate(None, None, c, None) kstart = 1 kstop = k elif op == 'start': c.set_key(k) self.session.truncate(None, c, None, None) kstart = k kstop = len(current) elif op == 'both': c2 = self.session.open_cursor(self.uri, None) # For both, the key given is the start key. Add 2 # for the stop key. kstart = k kstop = k + 2 c.set_key(kstart) c2.set_key(kstop) self.session.truncate(None, c, c2, None) c2.close() elif op == 'all': c2 = self.session.open_cursor(self.uri, None) kstart = 1 kstop = len(current) c.set_key(kstart) c2.set_key(kstop) self.session.truncate(None, c, c2, None) c2.close() while (kstart <= kstop): del current[kstart] kstart += 1 # print current # Check the state after each operation. self.check_all(current, committed) if txn == 'commit': committed = current.copy() self.session.commit_transaction() elif txn == 'rollback': current = committed.copy() self.session.rollback_transaction() # Check the state after each commit/rollback. self.check_all(current, committed) # Gather statistics - this needs to be done before the connection is # closed or statistics would be reset. stat_cursor = self.session.open_cursor('statistics:', None, None) clen = stat_cursor[stat.conn.log_compress_len][2] cmem = stat_cursor[stat.conn.log_compress_mem][2] cwrites = stat_cursor[stat.conn.log_compress_writes][2] cfails = stat_cursor[stat.conn.log_compress_write_fails][2] csmall = stat_cursor[stat.conn.log_compress_small][2] stat_cursor.close() if self.compress == '': self.assertEqual(clen, cmem) self.assertEqual(cwrites, 0) self.assertEqual(cfails, 0) elif self.compress == 'nop': self.assertEqual(clen, cmem) self.assertEqual(cwrites, 0) self.assertEqual((cfails > 0 or csmall > 0), True) else: self.assertEqual(clen < cmem, True) self.assertEqual(cwrites > 0, True) self.assertEqual((cfails > 0 or csmall > 0), True) # # Run printlog and make sure it exits with zero status. This should be # run as soon as we can after the crash to try and conflict with the # journal file read. # self.runWt(['-h', self.backup_dir, 'printlog'], outfilename='printlog.out')
class test_backup26(backup_base): dir = 'backup.dir' # Backup directory name uri = "table_backup" ntables = 10000 if wttest.islongtest() else 500 # Reverse the backup restore list, WiredTiger should still succeed in this case. reverse = [ ["reverse_target_list", dict(reverse=True)], ["target_list", dict(reverse=False)], ] # Percentage of tables to not copy over in selective backup. percentage = [ ('hundred_precent', dict(percentage=1)), ('ninety_percent', dict(percentage=0.9)), ('fifty_percent', dict(percentage=0.5)), ('ten_percent', dict(percentage=0.1)), ('zero_percent', dict(percentage=0)), ] scenarios = make_scenarios(percentage, reverse) def test_backup26(self): selective_remove_uri_file_list = [] selective_remove_uri_list = [] selective_uri_list = [] for i in range(0, self.ntables): uri = "table:{0}".format(self.uri + str(i)) dataset = SimpleDataSet(self, uri, 100, key_format="S") dataset.populate() # Append the table uri to the selective backup remove list until the set percentage. # These tables will not be copied over in selective backup. if (i <= int(self.ntables * self.percentage)): selective_remove_uri_list.append(uri) selective_remove_uri_file_list.append( "{0}.wt".format(self.uri + str(i))) else: selective_uri_list.append(uri) self.session.checkpoint() os.mkdir(self.dir) # Now copy the files using full backup. This should not include the tables inside the remove list. all_files = self.take_selective_backup(self.dir, selective_remove_uri_file_list) target_uris = None if self.reverse: target_uris = str(selective_uri_list[::-1]).replace("\'", "\"") else: target_uris = str(selective_uri_list).replace("\'", "\"") starttime = time.time() # After the full backup, open and recover the backup database. backup_conn = self.wiredtiger_open( self.dir, "backup_restore_target={0}".format(target_uris)) elapsed = time.time() - starttime self.pr("%s partial backup has taken %.2f seconds." % (str(self), elapsed)) bkup_session = backup_conn.open_session() # Open the cursor from uris that were not part of the selective backup and expect failure # since file doesn't exist. for remove_uri in selective_remove_uri_list: self.assertRaisesException( wiredtiger.WiredTigerError, lambda: bkup_session.open_cursor(remove_uri, None, None)) # Open the cursors on tables that copied over to the backup directory. They should still # recover properly. for uri in selective_uri_list: c = bkup_session.open_cursor(uri, None, None) ds = SimpleDataSet(self, uri, 100, key_format="S") ds.check_cursor(c) c.close() backup_conn.close()
class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): dir = 'backup.dir' # Backup directory name # This test is written to test LSM hot backups: we test a simple LSM object # and a complex LSM object, but we can't test them both at the same time # because we need to load fast enough the merge threads catch up, and so we # test the real database, not what the database might look like after the # merging settles down. # # The way it works is we create 4 objects, only one of which is large, then # we do a hot backup of one or more of the objects and compare the original # to the backup to confirm the backup is correct. pfx = 'test_backup' objs = [ # Objects ('table:' + pfx + '.1', SimpleDataSet, 0), ('lsm:' + pfx + '.2', SimpleDataSet, 1), ('table:' + pfx + '.3', ComplexDataSet, 2), ('table:' + pfx + '.4', ComplexLSMDataSet, 3), ] list = [ ('backup_1', dict(big=0, list=[0])), # Target objects individually ('backup_2', dict(big=1, list=[1])), ('backup_3', dict(big=2, list=[2])), ('backup_4', dict(big=3, list=[3])), ('backup_5a', dict(big=0, list=[0, 2])), # Target groups of objects ('backup_5b', dict(big=2, list=[0, 2])), ('backup_6a', dict(big=1, list=[1, 3])), ('backup_6b', dict(big=3, list=[1, 3])), ('backup_7a', dict(big=0, list=[0, 1, 2])), ('backup_7b', dict(big=1, list=[0, 1, 2])), ('backup_7c', dict(big=2, list=[0, 1, 2])), ('backup_8a', dict(big=0, list=[0, 1, 2, 3])), ('backup_8b', dict(big=1, list=[0, 1, 2, 3])), ('backup_8c', dict(big=2, list=[0, 1, 2, 3])), ('backup_8d', dict(big=3, list=[0, 1, 2, 3])), ('backup_9', dict(big=3, list=[])), # Backup everything ] scenarios = make_scenarios(list, prune=3, prunelong=1000) # Create a large cache, otherwise this test runs quite slowly. conn_config = 'cache_size=1G' # Populate a set of objects. def populate(self): for i in self.objs: if self.big == i[2]: rows = 200000 # Big object else: rows = 1000 # Small object i[1](self, i[0], rows).populate() # Backup needs a checkpoint self.session.checkpoint(None) # Compare the original and backed-up files using the wt dump command. def compare(self, uri): self.runWt(['dump', uri], outfilename='orig') self.runWt(['-h', self.dir, 'dump', uri], outfilename='backup') self.assertEqual(True, compare_files(self, 'orig', 'backup')) # Check that a URI doesn't exist, both the meta-data and the file names. def confirmPathDoesNotExist(self, uri): conn = self.wiredtiger_open(self.dir) session = conn.open_session() self.assertRaises(wiredtiger.WiredTigerError, lambda: session.open_cursor(uri, None, None)) conn.close() self.assertEqual( glob.glob(self.dir + '*' + uri.split(":")[1] + '*'), [], 'confirmPathDoesNotExist: URI exists, file name matching \"' + uri.split(":")[1] + '\" found') # Backup a set of target tables using a backup cursor. def backup_table_cursor(self, l): # Create the backup directory. os.mkdir(self.dir) # Build the target list. config = "" if l: config = 'target=(' for i in range(0, len(self.objs)): if i in l: config += '"' + self.objs[i][0] + '",' config += ')' # Open up the backup cursor, and copy the files. cursor = self.session.open_cursor('backup:', None, config) while True: ret = cursor.next() if ret != 0: break #print 'Copy from: ' + cursor.get_key() + ' to ' + self.dir shutil.copy(cursor.get_key(), self.dir) self.assertEqual(ret, wiredtiger.WT_NOTFOUND) cursor.close() # Confirm the objects we backed up exist, with correct contents. for i in range(0, len(self.objs)): if not l or i in l: self.compare(self.objs[i][0]) # Confirm the other objects don't exist. if l: for i in range(0, len(self.objs)): if i not in l: self.confirmPathDoesNotExist(self.objs[i][0]) # Test backup with targets. def test_backup_target(self): self.populate() self.backup_table_cursor(self.list)
class test_assert01(wttest.WiredTigerTestCase, suite_subprocess): base = 'assert01' base_uri = 'file:' + base uri_always = base_uri + '.always.wt' uri_def = base_uri + '.def.wt' uri_never = base_uri + '.never.wt' uri_none = base_uri + '.none.wt' cfg_always = 'verbose=[write_timestamp],write_timestamp_usage=always,assert=(write_timestamp=on)' cfg_def = '' cfg_never = 'verbose=(write_timestamp=true),write_timestamp_usage=never,assert=(write_timestamp=on)' cfg_none = 'assert=(write_timestamp=off)' key_format_values = [('column', dict(key_format='r', usestrings=False)), ('string-row', dict(key_format='S', usestrings=True))] scenarios = make_scenarios(key_format_values) count = 1 # # Commit a k/v pair making sure that it detects an error if needed, when # used with and without a commit timestamp. # def insert_check(self, uri, use_ts): c = self.session.open_cursor(uri) key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) # Commit with a timestamp self.session.begin_transaction() self.session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(self.count)) c[key] = val # All settings other than never should commit successfully if (use_ts != 'never'): self.session.commit_transaction() else: msg = "/timestamp set on this transaction/" self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: self.assertEquals( self.session.commit_transaction(), 0), msg) c.close() self.count += 1 # Commit without a timestamp key = 'key' + str(self.count) if self.usestrings else self.count val = 'value' + str(self.count) c = self.session.open_cursor(uri) self.session.begin_transaction() c[key] = val # All settings other than always should commit successfully if (use_ts != 'always'): self.session.commit_transaction() else: msg = "/none set on this transaction/" self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: self.assertEquals( self.session.commit_transaction(), 0), msg) self.count += 1 c.close() def test_commit_timestamp(self): cfg = 'key_format={},value_format=S,'.format(self.key_format) # Create a data item at a timestamp self.session.create(self.uri_always, cfg + self.cfg_always) self.session.create(self.uri_def, cfg + self.cfg_def) self.session.create(self.uri_never, cfg + self.cfg_never) self.session.create(self.uri_none, cfg + self.cfg_none) # Check inserting into each table self.insert_check(self.uri_always, 'always') self.insert_check(self.uri_def, 'none') self.insert_check(self.uri_never, 'never') self.insert_check(self.uri_none, 'none')
class test_rollback_to_stable26(test_rollback_to_stable_base): format_values = [ ('column', dict(key_format='r', value_format='S')), ('column_fix', dict(key_format='r', value_format='8t')), ('row_integer', dict(key_format='i', value_format='S')), ] hs_remove_values = [ ('no_hs_remove', dict(hs_remove=False)), ('hs_remove', dict(hs_remove=True)) ] prepare_remove_values = [ ('no_prepare_remove', dict(prepare_remove=False)), ('prepare_remove', dict(prepare_remove=True)) ] scenarios = make_scenarios(format_values, hs_remove_values, prepare_remove_values) def conn_config(self): config = 'cache_size=10MB,statistics=(all),timing_stress_for_test=[history_store_checkpoint_delay]' return config def evict_cursor(self, uri, nrows): # Configure debug behavior on a cursor to evict the page positioned on when the reset API is used. evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") self.session.begin_transaction("ignore_prepare=true") for i in range (1, nrows + 1): evict_cursor.set_key(i) evict_cursor.search() evict_cursor.reset() evict_cursor.close() self.session.rollback_transaction() def test_rollback_to_stable(self): nrows = 10 # Create a table without logging. uri = "table:rollback_to_stable26" ds = SimpleDataSet( self, uri, 0, key_format=self.key_format, value_format=self.value_format, config='log=(enabled=false)') ds.populate() if self.value_format == '8t': value_a = 97 value_b = 98 value_c = 99 value_d = 100 value_e = 101 else: value_a = "aaaaa" * 100 value_b = "bbbbb" * 100 value_c = "ccccc" * 100 value_d = "ddddd" * 100 value_e = "eeeee" * 100 # Pin oldest and stable to timestamp 10. self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) + ',stable_timestamp=' + self.timestamp_str(10)) self.large_updates(uri, value_a, ds, nrows, False, 20) self.large_updates(uri, value_b, ds, nrows, False, 30) if self.hs_remove: self.large_removes(uri, ds, nrows, False, 40) prepare_session = self.conn.open_session() prepare_session.begin_transaction() cursor = prepare_session.open_cursor(uri) for i in range (1, nrows + 1): cursor[i] = value_c if self.prepare_remove: cursor.set_key(i) self.assertEqual(cursor.remove(), 0) cursor.close() prepare_session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(50)) # Verify data is visible and correct. self.check(value_a, uri, nrows, None, 20) self.check(value_b, uri, nrows, None, 30) self.evict_cursor(uri, nrows) # Pin stable to timestamp 40. self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40)) # Create a checkpoint thread done = threading.Event() ckpt = checkpoint_thread(self.conn, done) try: ckpt.start() # Sleep for sometime so that checkpoint starts before committing last transaction. time.sleep(5) prepare_session.rollback_transaction() finally: done.set() ckpt.join() self.large_updates(uri, value_d, ds, nrows, False, 60) # Check that the correct data. self.check(value_a, uri, nrows, None, 20) self.check(value_b, uri, nrows, None, 30) self.check(value_d, uri, nrows, None, 60) # Simulate a server crash and restart. simulate_crash_restart(self, ".", "RESTART") stat_cursor = self.session.open_cursor('statistics:', None, None) hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2] hs_restore_updates = stat_cursor[stat.conn.txn_rts_hs_restore_updates][2] keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2] stat_cursor.close() self.assertEqual(keys_removed, 0) self.assertEqual(hs_restore_updates, nrows) self.assertEqual(hs_removed, nrows) # Check that the correct data. self.check(value_a, uri, nrows, None, 20) self.check(value_b, uri, nrows, None, 30) self.large_updates(uri, value_e, ds, nrows, False, 60) self.evict_cursor(uri, nrows) # Check that the correct data. self.check(value_a, uri, nrows, None, 20) self.check(value_b, uri, nrows, None, 30) self.check(value_e, uri, nrows, None, 60)
class test_compat01(wttest.WiredTigerTestCase, suite_subprocess): # Add enough entries and use a small log size to generate more than # one log file. entries = 2000 logmax = "100K" tablename = 'test_compat01' uri = 'table:' + tablename # Declare the log versions that do and do not have prevlsn. # Log version 1 does not have the prevlsn record. # Log version 2 introduced that record. # Log version 3 continues to have that record. min_logv = 2 latest_logv = 3 # The API uses only the major and minor numbers but accepts with # and without the patch number. Test both. start_compat = [ ('def', dict(compat1='none', logv1=3)), ('31', dict(compat1="3.1", logv1=3)), ('30', dict(compat1="3.0", logv1=2)), ('30_patch', dict(compat1="3.0.0", logv1=2)), ('26', dict(compat1="2.6", logv1=1)), ('old', dict(compat1="1.8", logv1=1)), ('old_patch', dict(compat1="1.8.1", logv1=1)), ] restart_compat = [ ('def2', dict(compat2='none', logv2=3)), ('31_2', dict(compat2="3.1", logv2=3)), ('30_2', dict(compat2="3.0", logv2=2)), ('30_patch2', dict(compat2="3.0.0", logv2=2)), ('26_2', dict(compat2="2.6", logv2=1)), ('old2', dict(compat2="1.8", logv2=1)), ('old_patch2', dict(compat2="1.8.1", logv2=1)), ] scenarios = make_scenarios(restart_compat, start_compat) def make_compat_str(self, create): compat_str = '' if (create == True and self.compat1 != 'none'): #compat_str = 'verbose=(temporary),compatibility=(release="%s"),' % self.compat1 compat_str = 'compatibility=(release="%s"),' % self.compat1 elif (create == False and self.compat2 != 'none'): #compat_str = 'verbose=(temporary),compatibility=(release="%s"),' % self.compat2 compat_str = 'compatibility=(release="%s"),' % self.compat2 return compat_str def conn_config(self): # Set archive false on the home directory. log_str = 'log=(archive=false,enabled,file_max=%s),' % self.logmax compat_str = self.make_compat_str(True) self.pr("Conn config:" + log_str + compat_str) return log_str + compat_str def check_prev_lsn(self, exists, conn_close): # # Run printlog and look for the prev_lsn log record. Verify its # existence with the passed in expected result. We don't use # check_file_contains because that only looks in the first 100K and # we don't know how big our text-based log output is. Look through # the entire file if needed and set a boolean for comparison. # self.runWt(['printlog'], outfilename='printlog.out', closeconn=conn_close) contains = False with open('printlog.out') as logfile: for line in logfile: if 'optype' in line and 'prev_lsn' in line: contains = True break self.assertEqual(exists, contains) def check_log(self, reconfig): orig_logs = fnmatch.filter(os.listdir('.'), "*gerLog*") compat_str = self.make_compat_str(False) if not reconfig: # # Close and open the connection to force recovery and log archiving # even if archive is turned off (in some circumstances). If we are # downgrading we must archive newer logs. Verify the log files # have or have not been archived. # exist = True if self.logv1 < self.min_logv: exist = False self.check_prev_lsn(exist, True) self.conn.close() log_str = 'log=(enabled,file_max=%s,archive=false),' % self.logmax restart_config = log_str + compat_str self.pr("Restart conn " + restart_config) # # Open a connection to force it to run recovery. # conn = self.wiredtiger_open('.', restart_config) conn.close() check_close = False else: self.pr("Reconfigure: " + compat_str) self.conn.reconfigure(compat_str) check_close = True # # Archiving is turned off explicitly. # # Check logs. The original logs should have been archived only if # we downgraded. In all other cases the original logs should be there. # Downgrade means not running the latest possible log version, not # the difference between original and current. cur_logs = fnmatch.filter(os.listdir('.'), "*Log*") log_present = True if self.logv1 != self.logv2 and self.logv2 != self.latest_logv: log_present = False for o in orig_logs: self.assertEqual(log_present, o in cur_logs) # Run printlog and verify the new record does or does not exist. exist = True if self.logv2 < self.min_logv: exist = False self.check_prev_lsn(exist, check_close) def run_test(self, reconfig): # If reconfiguring with the empty string there is nothing to do. if reconfig == True and self.compat2 == 'none': return self.session.create(self.uri, 'key_format=i,value_format=i') c = self.session.open_cursor(self.uri, None) # # Add some entries to generate log files. # for i in range(self.entries): c[i] = i + 1 c.close() # Check the log state after the entire op completes # and run recovery with the restart compatibility mode. self.check_log(reconfig) # Run the same test but reset the compatibility via # reconfigure or changing it when reopening the connection. def test_reconfig(self): self.run_test(True) def test_restart(self): self.run_test(False)