def test_truncate_cursor_order(self):
uri = self.type + self.name
# A simple, one-file file or table object.
ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt)
ds.populate()
c1 = self.session.open_cursor(uri, None)
c1.set_key(ds.key(1000))
c2 = self.session.open_cursor(uri, None)
c2.set_key(ds.key(2000))
self.session.truncate(None, c1, c2, None)
self.assertEqual(c1.close(), 0)
self.assertEqual(c2.close(), 0)
self.session.drop(uri)
if self.type == "table:":
ds = ComplexDataSet(self, uri, 100, key_format=self.keyfmt)
ds.populate()
c1 = self.session.open_cursor(uri, None)
c1.set_key(ds.key(1000))
c2 = self.session.open_cursor(uri, None)
c2.set_key(ds.key(2000))
self.session.truncate(None, c1, c2, None)
self.assertEqual(c1.close(), 0)
self.assertEqual(c2.close(), 0)
self.session.drop(uri)
def test_prepare_lookaside(self):
if not wiredtiger.timestamp_build():
self.skipTest('requires a timestamp build')
# Create a small table.
uri = "table:test_prepare_lookaside01"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u')
ds.populate()
bigvalue = "aaaaa" * 100
# Initially load huge data
cursor = self.session.open_cursor(uri)
for i in range(1, 10000):
cursor.set_key(ds.key(nrows + i))
cursor.set_value(bigvalue)
self.assertEquals(cursor.insert(), 0)
cursor.close()
self.session.checkpoint()
# Check if lookaside is working properly with prepare transactions.
# We put prepared updates in multiple sessions so that we do not hang
# because of cache being full with uncommitted updates.
nsessions = 3
nkeys = 4000
self.prepare_updates(uri, ds, nrows, nsessions, nkeys)
def test_drop(self):
uri = 'lsm:' + self.name
ds = SimpleDataSet(self, uri, 100000)
ds.populate()
self.reopen_conn()
self.session.drop(uri, None)
def test_insert_over_delete_replace(self):
msg = '/WT_CACHE_FULL.*/'
ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt,
value_format=self.valuefmt, config=self.table_config)
self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
ds.populate, msg)
cursor = self.session.open_cursor(self.uri, None)
cursor.prev()
last_key = int(cursor.get_key())
# Now that the database contains as much data as will fit into
# the configured cache, verify removes succeed.
cursor = self.session.open_cursor(self.uri, None)
for i in range(1, last_key / 4, 1):
cursor.set_key(ds.key(i))
cursor.remove()
cursor.reset()
# Spin inserting to give eviction a chance to reclaim space
inserted = False
for i in range(1, 1000):
try:
cursor[ds.key(1)] = ds.value(1)
except wiredtiger.WiredTigerError:
cursor.reset()
sleep(1)
continue
inserted = True
break
self.assertTrue(inserted)
def test_modify_smoke_recover(self):
# Close the original database.
self.conn.close()
# Open a new database with logging configured.
self.conn_config = \
'log=(enabled=true),transaction_sync=(method=dsync,enabled)'
self.conn = self.setUpConnectionOpen(".")
self.session = self.setUpSessionOpen(self.conn)
# Populate a database, and checkpoint it so it exists after recovery.
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.session.checkpoint()
self.modify_load(ds, False)
# Crash and recover in a new directory.
newdir = 'RESTART'
copy_wiredtiger_home('.', newdir)
self.conn.close()
self.conn = self.setUpConnectionOpen(newdir)
self.session = self.setUpSessionOpen(self.conn)
self.session.verify(self.uri)
self.modify_confirm(ds, False)
def test_basic(self):
ds = SimpleDataSet(self, self.uri, self.nentries,
config=self.config, key_format=self.keyfmt)
ds.populate()
self.reopen_conn()
c = self.session.open_cursor(self.uri, None)
self.forward(c, ds, self.nentries, [])
self.backward(c, ds, self.nentries, [])
def test_modify_smoke_single(self):
if self.skip():
return
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format='u')
ds.populate()
self.modify_load(ds, True)
def test_modify_smoke_reopen(self):
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.modify_load(ds, False)
# Flush to disk, forcing reconciliation.
self.reopen_conn()
self.modify_confirm(ds, False)
def test_checkpoint_target(self):
# Create 3 objects, change one record to an easily recognizable string.
uri = self.uri + '1'
ds1 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
ds1.populate()
self.update(uri, ds1, 'ORIGINAL')
uri = self.uri + '2'
ds2 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
ds2.populate()
self.update(uri, ds2, 'ORIGINAL')
uri = self.uri + '3'
ds3 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
ds3.populate()
self.update(uri, ds3, 'ORIGINAL')
# Checkpoint all three objects.
self.session.checkpoint("name=checkpoint-1")
# Update all 3 objects, then checkpoint two of the objects with the
# same checkpoint name.
self.update(self.uri + '1', ds1, 'UPDATE')
self.update(self.uri + '2', ds2, 'UPDATE')
self.update(self.uri + '3', ds3, 'UPDATE')
target = 'target=("' + self.uri + '1"' + ',"' + self.uri + '2")'
self.session.checkpoint("name=checkpoint-1," + target)
# Confirm the checkpoint has the old value in objects that weren't
# checkpointed, and the new value in objects that were checkpointed.
self.check(self.uri + '1', ds1, 'UPDATE')
self.check(self.uri + '2', ds2, 'UPDATE')
self.check(self.uri + '3', ds3, 'ORIGINAL')
def test_checkpoint_stats(self):
ds = SimpleDataSet(self, self.uri, self.nentries,
config=self.config, key_format=self.keyfmt)
for name in ('first', 'second', 'third'):
ds.populate()
self.session.checkpoint('name=' + name)
cursor = self.session.open_cursor(
'statistics:' + self.uri, None, 'checkpoint=' + name)
self.assertEqual(
cursor[stat.dsrc.btree_entries][2], self.nentries)
cursor.close()
def test_insert_over_delete(self):
msg = '/WT_CACHE_FULL.*/'
ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt,
value_format=self.valuefmt, config=self.table_config)
self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
ds.populate, msg)
# Now that the database contains as much data as will fit into
# the configured cache, verify removes succeed.
cursor = self.session.open_cursor(self.uri, None)
for i in range(1, 100):
cursor.set_key(ds.key(i))
cursor.remove()
def test_truncate_cursor_order(self):
uri = self.type + self.name
ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt)
ds.populate()
c1 = self.session.open_cursor(uri, None)
c2 = self.session.open_cursor(uri, None)
c1.set_key(ds.key(20))
c2.set_key(ds.key(10))
msg = "/the start cursor position is after the stop cursor position/"
self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg)
c2.set_key(ds.key(20))
self.session.truncate(None, c1, c2, None)
def test_search_empty(self):
# Create the object and open a cursor.
ds = SimpleDataSet(self, self.uri, 0, key_format=self.key_format,
value_format=self.value_format)
ds.create()
cursor = self.session.open_cursor(self.uri, None)
# Search for a record past the end of the table, which should fail.
cursor.set_key(ds.key(100))
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
# Search-near for a record past the end of the table, which should fail.
cursor.set_key(ds.key(100))
self.assertEqual(cursor.search_near(), wiredtiger.WT_NOTFOUND)
def test_truncate_cursor_notset(self):
uri = self.type + self.name
msg = "/requires key be set/"
ds = SimpleDataSet(self, uri, 100)
ds.populate()
c1 = self.session.open_cursor(uri, None)
c2 = self.session.open_cursor(uri, None)
c2.set_key(ds.key(10))
self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg)
self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c2, c1, None), msg)
c1.close()
c2.close()
def test_las(self):
# Create a small table.
uri = "table:test_las"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format="S")
ds.populate()
bigvalue = "aaaaa" * 100
# Initially load huge data
cursor = self.session.open_cursor(uri)
for i in range(1, 10000):
cursor.set_key(ds.key(nrows + i))
cursor.set_value(bigvalue)
self.assertEquals(cursor.insert(), 0)
cursor.close()
self.session.checkpoint()
# Scenario: 1
# Check to see LAS working with old snapshot
bigvalue1 = "bbbbb" * 100
self.session.snapshot("name=xxx")
# Update the values in different session after snapshot
self.large_updates(self.session, uri, bigvalue1, ds, nrows)
# Check to see the value after recovery
self.durable_check(bigvalue1, uri, ds, nrows)
self.session.snapshot("drop=(all)")
# Scenario: 2
# Check to see LAS working with old reader
bigvalue2 = "ccccc" * 100
session2 = self.conn.open_session()
session2.begin_transaction('isolation=snapshot')
self.large_updates(self.session, uri, bigvalue2, ds, nrows)
# Check to see the value after recovery
self.durable_check(bigvalue2, uri, ds, nrows)
session2.rollback_transaction()
session2.close()
# Scenario: 3
# Check to see LAS working with old timestamp
bigvalue3 = "ddddd" * 100
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1))
self.large_updates(self.session, uri, bigvalue3, ds, nrows, timestamp=True)
# Check to see data can be see only till the stable_timestamp
self.durable_check(bigvalue2, uri, ds, nrows)
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(i + 1))
# Check to see latest data can be seen
self.durable_check(bigvalue3, uri, ds, nrows)
def test_reconfig_fail(self):
uri = 'table:reconfig_fail'
ds = SimpleDataSet(self, uri, 100, key_format='S')
ds.populate()
self.session.begin_transaction("isolation=snapshot")
c = self.session.open_cursor(uri, None)
c.set_key(ds.key(20))
c.set_value("abcde")
self.assertEquals(c.update(), 0)
compat_str = 'compatibility=(release="3.0.0")'
msg = '/system must be quiescent/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda:self.conn.reconfigure(compat_str), msg)
def test_modify_delete(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format='u')
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
self.assertEquals(c.remove(), 0)
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(10))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
def test_checkpoint_cursor_update(self):
ds = SimpleDataSet(self, self.uri, 100, key_format=self.fmt)
ds.populate()
self.session.checkpoint("name=ckpt")
cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
cursor.set_key(ds.key(10))
cursor.set_value("XXX")
msg = "/Unsupported cursor/"
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cursor.insert(), msg)
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cursor.remove(), msg)
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cursor.update(), msg)
cursor.close()
def test_insert_over_capacity(self):
msg = '/WT_CACHE_FULL.*/'
ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt,
value_format=self.valuefmt, config=self.table_config)
self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
ds.populate, msg)
# Figure out the last key we successfully inserted, and check all
# previous inserts are still there.
cursor = self.session.open_cursor(self.uri, None)
cursor.prev()
last_key = int(cursor.get_key())
ds = SimpleDataSet(self, self.uri, last_key, key_format=self.keyfmt,
value_format=self.valuefmt, config=self.table_config)
ds.check()
def test_insert_over_allowed(self):
# Create a new table that is allowed to exceed the cache size, do this
# before filling the cache so that the create succeeds
self.session.create(self.uri + "_over", "ignore_in_memory_cache_size=true")
# Populate a table with enough data to fill the cache.
msg = "/WT_CACHE_FULL.*/"
ds = SimpleDataSet(self, self.uri, 10000000, config=self.table_config)
self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, lambda: ds.populate(), msg)
# Add some content to the new table
cursor = self.session.open_cursor(self.uri + "_over", None)
for i in range(1, 1000):
cursor[str("%015d" % i)] = str(i) + ": abcdefghijklmnopqrstuvwxyz"
cursor.close()
def test_hazard(self):
uri = "table:hazard"
ds = SimpleDataSet(self, uri, 1000)
ds.populate()
# Open 10,000 cursors and pin a page to set a hazard pointer.
cursors = []
for i in range(0, 10000):
c = self.session.open_cursor(uri, None)
c.set_key(ds.key(10))
c.search()
cursors.append(c)
# Close the cursors, clearing the hazard pointer.
for c in cursors:
c.close()
def test_modify_abort(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
# Start a transaction.
self.session.begin_transaction()
# Insert a new record.
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(30))
c.set_value(ds.value(30))
self.assertEquals(c.insert(), 0)
# Test that we can successfully modify our own record.
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), 0)
# Test that another transaction cannot modify our uncommitted record.
xs = self.conn.open_session()
xc = xs.open_cursor(self.uri, None)
xs.begin_transaction()
xc.set_key(ds.key(30))
xc.set_value(ds.value(30))
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
xc.set_key(ds.key(30))
self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND)
xs.rollback_transaction()
# Rollback our transaction.
self.session.rollback_transaction()
# Test that we can't modify our aborted insert.
self.session.begin_transaction()
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
self.session.rollback_transaction()
def test_las(self):
# Create a small table.
uri = "table:test_las"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format="S")
ds.populate()
# Take a snapshot.
self.session.snapshot("name=xxx")
# Insert a large number of records, we'll hang if the lookaside table
# isn't doing its thing.
c = self.session.open_cursor(uri)
bigvalue = "abcde" * 100
for i in range(1, 1000000):
c.set_key(ds.key(nrows + i))
c.set_value(bigvalue)
self.assertEquals(c.insert(), 0)
def test_smoke(self):
ds = SimpleDataSet(self, self.uri, self.nentries,
config=self.config, key_format=self.keyfmt)
ds.populate()
self.reopen_conn()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(100))
self.assertEqual(c.search(), 0)
self.assertEqual(c.get_value(), ds.value(100))
c.set_key(ds.key(101))
self.assertEqual(c.search(), 0)
self.assertEqual(c.get_value(), ds.value(101))
c.set_key(ds.key(9999))
self.assertEqual(c.search(), 0)
self.assertEqual(c.get_value(), ds.value(9999))
def test_cursor_random_reasonable_distribution(self):
uri = self.type
num_entries = self.records
if uri == 'table:random':
config = 'leaf_page_max=100MB'
else:
config = ''
# Set the leaf-page-max value, otherwise the page might split.
ds = SimpleDataSet(self, uri, num_entries, config=config)
ds.populate()
# Setup an array to track which keys are seen
visitedKeys = [0] * (num_entries + 1)
# Setup a counter to see when we find a sequential key
sequentialKeys = 0
cursor = self.session.open_cursor(uri, None, 'next_random=true')
lastKey = None
for i in range(0, num_entries):
self.assertEqual(cursor.next(), 0)
current = cursor.get_key()
current = int(current)
visitedKeys[current] = visitedKeys[current] + 1
if lastKey != None:
if current == (lastKey + 1):
sequentialKeys += 1
lastKey = current
differentKeys = sum(x > 0 for x in visitedKeys)
#print visitedKeys
#print differentKeys
'''
self.tty('differentKeys: ' + str(differentKeys) + ' of ' + \
str(num_entries) + ', ' + \
str((int)((differentKeys * 100) / num_entries)) + '%')
'''
# Can't test for non-sequential data when there is 1 item in the table
if num_entries > 1:
self.assertGreater(num_entries - 1, sequentialKeys,
'cursor is returning sequential data')
self.assertGreater(differentKeys, num_entries / 4,
'next_random random distribution not adequate')
def test_checkpoint_las_reads(self):
if not wiredtiger.timestamp_build():
self.skipTest('requires a timestamp build')
# Create a small table.
uri = "table:test_las03"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u')
ds.populate()
bigvalue = "aaaaa" * 100
# Initially load huge data
cursor = self.session.open_cursor(uri)
for i in range(1, 10000):
cursor[ds.key(nrows + i)] = bigvalue
cursor.close()
self.session.checkpoint()
# Check to see LAS working with old timestamp
bigvalue2 = "ddddd" * 100
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1))
las_writes_start = self.get_stat(stat.conn.cache_write_lookaside)
self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000)
# If the test sizing is correct, the history will overflow the cache
self.session.checkpoint()
las_writes = self.get_stat(stat.conn.cache_write_lookaside) - las_writes_start
self.assertGreaterEqual(las_writes, 0)
for ts in range(2, 4):
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(ts))
# Now just update one record and checkpoint again
self.large_updates(self.session, uri, bigvalue2, ds, nrows, 1)
las_reads_start = self.get_stat(stat.conn.cache_read_lookaside)
self.session.checkpoint()
las_reads = self.get_stat(stat.conn.cache_read_lookaside) - las_reads_start
# Since we're dealing with eviction concurrent with checkpoints
# and skewing is controlled by a heuristic, we can't put too tight
# a bound on this.
self.assertLessEqual(las_reads, 100)
def test_modify_many(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format='u')
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
orig = 'abcdefghijklmnopqrstuvwxyz'
c.set_value(orig)
self.assertEquals(c.update(), 0)
for i in range(0, 50000):
new = "".join([random.choice(string.digits) for i in xrange(5)])
orig = orig[:10] + new + orig[15:]
mods = []
mod = wiredtiger.Modify(new, 10, 5)
mods.append(mod)
self.assertEquals(c.modify(mods), 0)
c.set_key(ds.key(10))
self.assertEquals(c.search(), 0)
self.assertEquals(c.get_value(), orig)
def test_search_duplicate(self):
if self.colvar == 0:
return
# Populate the tree.
ds = SimpleDataSet(self, self.uri, 105, key_format=self.key_format,
value_format=self.value_format)
ds.populate()
# Set up deleted records before and after a set of duplicate records,
# and make sure search/search-near returns the correct record.
cursor = self.session.open_cursor(self.uri, None)
for i in range(20, 100):
cursor[ds.key(i)] = '=== IDENTICAL VALUE ==='
for i in range(15, 25):
cursor.set_key(ds.key(i))
self.assertEqual(cursor.remove(), 0)
for i in range(95, 106):
cursor.set_key(ds.key(i))
self.assertEqual(cursor.remove(), 0)
cursor.close()
# Reopen the connection, forcing it to disk and moving the records to
# an on-page format.
self.reopen_conn()
# Open a cursor.
cursor = self.session.open_cursor(self.uri, None)
# Search-near for a record in the deleted set before the duplicate set,
# which should succeed, returning the first record in the duplicate set.
cursor.set_key(ds.key(18))
self.assertEqual(cursor.search_near(), 1)
self.assertEqual(cursor.get_key(), ds.key(25))
# Search-near for a record in the deleted set after the duplicate set,
# which should succeed, returning the last record in the duplicate set.
cursor.set_key(ds.key(98))
self.assertEqual(cursor.search_near(), -1)
self.assertEqual(cursor.get_key(), ds.key(94))
def test_checkpoint_illegal_name(self):
ds = SimpleDataSet(self, "file:checkpoint", 100, key_format='S')
ds.populate()
msg = '/the checkpoint name.*is reserved/'
for conf in (
'name=WiredTigerCheckpoint',
'name=WiredTigerCheckpoint.',
'name=WiredTigerCheckpointX',
'drop=(from=WiredTigerCheckpoint)',
'drop=(from=WiredTigerCheckpoint.)',
'drop=(from=WiredTigerCheckpointX)',
'drop=(to=WiredTigerCheckpoint)',
'drop=(to=WiredTigerCheckpoint.)',
'drop=(to=WiredTigerCheckpointX)'):
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.session.checkpoint(conf), msg)
msg = '/WiredTiger objects should not include grouping/'
for conf in (
'name=check{point',
'name=check\\point'):
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.session.checkpoint(conf), msg)
def test_checkpoint_last(self):
# Create an object, change one record to an easily recognizable string,
# then checkpoint it and open a cursor, confirming we see the correct
# value. Repeat this action, we want to be sure the engine gets the
# latest checkpoint information each time.
uri = self.uri
ds = SimpleDataSet(self, uri, 100, key_format=self.fmt)
ds.populate()
for value in ('FIRST', 'SECOND', 'THIRD', 'FOURTH', 'FIFTH'):
# Update the object.
cursor = self.session.open_cursor(uri, None, "overwrite")
cursor[ds.key(10)] = value
cursor.close()
# Checkpoint the object.
self.session.checkpoint()
# Verify the "last" checkpoint sees the correct value.
cursor = self.session.open_cursor(
uri, None, "checkpoint=WiredTigerCheckpoint")
self.assertEquals(cursor[ds.key(10)], value)
def test_checkpoint_snapshot(self):
self.moresetup()
ds = SimpleDataSet(self, self.uri, 0, \
key_format=self.key_format, value_format=self.value_format, \
config='log=(enabled=false)'+self.extraconfig)
ds.populate()
cursor = self.session.open_cursor(self.uri, None, "bulk")
for i in range(1, self.nrows + 1):
if self.value_format == '8t':
cursor[i] = self.valuea
else:
cursor[i] = self.valuea + str(i)
cursor.close()
self.check(self.valuea, self.uri, self.nrows)
session1 = self.conn.open_session()
session1.begin_transaction()
cursor1 = session1.open_cursor(self.uri)
for i in range(1, self.nrows + 1):
cursor1.set_key(ds.key(i))
if self.value_format == '8t':
cursor1.set_value(self.valueb)
else:
cursor1.set_value(self.valueb + str(i))
self.assertEqual(cursor1.update(), 0)
# Create a checkpoint thread
done = threading.Event()
ckpt = checkpoint_thread(self.conn, done)
try:
ckpt.start()
# Wait for checkpoint to start before committing.
ckpt_started = 0
while not ckpt_started:
stat_cursor = self.session.open_cursor('statistics:', None,
None)
ckpt_started = stat_cursor[stat.conn.txn_checkpoint_running][2]
stat_cursor.close()
time.sleep(1)
session1.commit_transaction()
self.evict(self.uri, ds, self.nrows)
finally:
done.set()
ckpt.join()
#Take a backup and restore it.
self.take_full_backup(".", self.backup_dir)
self.reopen_conn(self.backup_dir)
# Check the table contains the last checkpointed value.
self.check(self.valuea, self.uri, self.nrows)
stat_cursor = self.session.open_cursor('statistics:', None, None)
inconsistent_ckpt = stat_cursor[stat.conn.txn_rts_inconsistent_ckpt][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
stat_cursor.close()
self.assertGreater(inconsistent_ckpt, 0)
self.assertEqual(keys_removed, 0)
def test_rollback_to_stable(self):
nrows = 1500
# Prepare transactions for column store table is not yet supported.
if self.prepare and self.key_format == 'r':
self.skipTest('Prepare transactions for column store table is not yet supported')
# Create a table without logging.
self.pr("create/populate table")
uri = "table:rollback_to_stable14"
ds = SimpleDataSet(
self, uri, 0, key_format=self.key_format, value_format="S", config='log=(enabled=false)')
ds.populate()
# Pin oldest and stable to timestamp 10.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) +
',stable_timestamp=' + timestamp_str(10))
value_a = "aaaaa" * 100
value_modQ = mod_val(value_a, 'Q', 0)
value_modR = mod_val(value_modQ, 'R', 1)
value_modS = mod_val(value_modR, 'S', 2)
value_modT = mod_val(value_modS, 'T', 3)
# Perform a combination of modifies and updates.
self.pr("large updates and modifies")
self.large_updates(uri, value_a, ds, nrows, self.prepare, 20)
self.large_modifies(uri, 'Q', ds, 0, 1, nrows, self.prepare, 30)
self.large_modifies(uri, 'R', ds, 1, 1, nrows, self.prepare, 40)
self.large_modifies(uri, 'S', ds, 2, 1, nrows, self.prepare, 50)
self.large_modifies(uri, 'T', ds, 3, 1, nrows, self.prepare, 60)
# Verify data is visible and correct.
self.check(value_a, uri, nrows, 20)
self.check(value_modQ, uri, nrows, 30)
self.check(value_modR, uri, nrows, 40)
self.check(value_modS, uri, nrows, 50)
self.check(value_modT, uri, nrows, 60)
# Pin stable to timestamp 60 if prepare otherwise 50.
if self.prepare:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60))
else:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50))
# Create a checkpoint thread
done = threading.Event()
ckpt = checkpoint_thread(self.conn, done)
try:
self.pr("start checkpoint")
ckpt.start()
# Perform several modifies in parallel with checkpoint.
# Rollbacks may occur when checkpoint is running, so retry as needed.
self.pr("modifies")
retry_rollback(self, 'modify ds1, W', None,
lambda: self.large_modifies(uri, 'W', ds, 4, 1, nrows, self.prepare, 70))
retry_rollback(self, 'modify ds1, X', None,
lambda: self.large_modifies(uri, 'X', ds, 5, 1, nrows, self.prepare, 80))
retry_rollback(self, 'modify ds1, Y', None,
lambda: self.large_modifies(uri, 'Y', ds, 6, 1, nrows, self.prepare, 90))
retry_rollback(self, 'modify ds1, Z', None,
lambda: self.large_modifies(uri, 'Z', ds, 7, 1, nrows, self.prepare, 100))
finally:
done.set()
ckpt.join()
# Simulate a server crash and restart.
self.pr("restart")
simulate_crash_restart(self, ".", "RESTART")
self.pr("restart complete")
stat_cursor = self.session.open_cursor('statistics:', None, None)
calls = stat_cursor[stat.conn.txn_rts][2]
hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2]
hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2]
hs_restore_updates = stat_cursor[stat.conn.txn_rts_hs_restore_updates][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2]
pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2]
upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2]
stat_cursor.close()
self.assertEqual(calls, 0)
self.assertEqual(keys_removed, 0)
self.assertEqual(hs_restore_updates, nrows)
self.assertEqual(keys_restored, 0)
if self.prepare:
self.assertGreaterEqual(upd_aborted, 0)
else:
self.assertEqual(upd_aborted, 0)
self.assertGreater(pages_visited, 0)
self.assertGreaterEqual(hs_removed, nrows)
self.assertGreaterEqual(hs_sweep, 0)
# Check that the correct data is seen at and after the stable timestamp.
self.check(value_a, uri, nrows, 20)
self.check(value_modQ, uri, nrows, 30)
self.check(value_modR, uri, nrows, 40)
self.check(value_modS, uri, nrows, 50)
# The test may output the following message in eviction under cache pressure. Ignore that.
self.ignoreStdoutPatternIfExists("oldest pinned transaction ID rolled back for eviction")
def test_rollback_to_stable(self):
nrows = 1000000
# Create a table.
uri = "table:rollback_to_stable12"
ds = SimpleDataSet(self,
uri,
0,
key_format=self.key_format,
value_format=self.value_format,
config='split_pct=50')
ds.populate()
if self.value_format == '8t':
value_a = 97
value_b = 98
else:
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
# Pin oldest and stable to timestamp 10.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) +
',stable_timestamp=' + self.timestamp_str(10))
# Perform several updates.
self.large_updates(uri, value_a, ds, nrows, self.prepare, 20)
# Verify data is visible and correct.
self.check(value_a, uri, nrows, None, 20)
# Pin stable to timestamp 30 if prepare otherwise 20.
if self.prepare:
self.conn.set_timestamp('stable_timestamp=' +
self.timestamp_str(30))
else:
self.conn.set_timestamp('stable_timestamp=' +
self.timestamp_str(20))
# Load a single row modification to be removed.
commit_ts = 30
cursor = self.session.open_cursor(uri)
self.session.begin_transaction()
cursor[ds.key(1)] = value_b
if self.prepare:
self.session.prepare_transaction('prepare_timestamp=' +
self.timestamp_str(commit_ts - 1))
self.session.timestamp_transaction('commit_timestamp=' +
self.timestamp_str(commit_ts))
self.session.timestamp_transaction('durable_timestamp=' +
self.timestamp_str(commit_ts +
1))
self.session.commit_transaction()
else:
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(commit_ts))
cursor.close()
self.session.checkpoint()
# Simulate a server crash and restart.
simulate_crash_restart(self, ".", "RESTART")
# Check that the correct data is seen at and after the stable timestamp.
self.check(value_a, uri, nrows, None, 30)
stat_cursor = self.session.open_cursor('statistics:', None, None)
calls = stat_cursor[stat.conn.txn_rts][2]
hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2]
hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2]
pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2]
pages_walk_skipped = stat_cursor[
stat.conn.txn_rts_tree_walk_skip_pages][2]
upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2]
stat_cursor.close()
self.assertEqual(calls, 0)
self.assertEqual(keys_removed, 0)
self.assertEqual(keys_restored, 0)
self.assertGreaterEqual(upd_aborted, 0)
self.assertGreater(pages_visited, 0)
self.assertGreaterEqual(hs_removed, 0)
self.assertEqual(hs_sweep, 0)
self.assertGreaterEqual(pages_walk_skipped, 0)
def test_timestamp(self):
uri = "table:test_timestamp19"
create_params = 'value_format=S,key_format=i'
self.session.create(uri, create_params)
ds = SimpleDataSet(self,
uri,
0,
key_format="i",
value_format="S",
config='log=(enabled=false)')
ds.populate()
nrows = 1000
value_x = 'x' * 1000
value_y = 'y' * 1000
value_z = 'z' * 1000
# Set the oldest and stable timestamps to 10.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) +
', stable_timestamp=' + timestamp_str(10))
# Insert values with varying timestamps.
self.updates(uri, value_x, ds, nrows, 20)
self.updates(uri, value_y, ds, nrows, 30)
self.updates(uri, value_z, ds, nrows, 40)
# Perform a checkpoint.
self.session.checkpoint('use_timestamp=true')
# Move the oldest and stable timestamps to 40.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(40) +
', stable_timestamp=' + timestamp_str(40))
# Update values.
self.updates(uri, value_z, ds, nrows, 50)
self.updates(uri, value_x, ds, nrows, 60)
self.updates(uri, value_y, ds, nrows, 70)
# Perform a checkpoint.
self.session.checkpoint('use_timestamp=true')
# Close and reopen the connection.
self.close_conn()
self.conn = self.setUpConnectionOpen('.')
self.session = self.setUpSessionOpen(self.conn)
# The oldest timestamp on recovery is 40. Trying to set it earlier is a no-op.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10))
self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'),
timestamp_str(40))
# Trying to set an earlier stable timestamp is an error.
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: self.conn.set_timestamp(
'stable_timestamp=' + timestamp_str(10)),
'/oldest timestamp \(0, 40\) must not be later than stable timestamp \(0, 10\)/'
)
self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'),
timestamp_str(40))
# Move the oldest and stable timestamps to 70.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(70) +
', stable_timestamp=' + timestamp_str(70))
self.assertTimestampsEqual(self.conn.query_timestamp('get=oldest'),
timestamp_str(70))
self.assertTimestampsEqual(self.conn.query_timestamp('get=stable'),
timestamp_str(70))
def test_readonly(self):
create_params = 'key_format=i,value_format=i'
entries = 10
# Create a database and a table.
SimpleDataSet(self,
self.uri,
entries,
key_format='i',
value_format='i').populate()
#
# Now close and reopen. Note that the connection function
# above will reopen it readonly.
self.reopen_conn()
msg = '/Unsupported/'
c = self.session.open_cursor(self.uri, None, None)
for op in self.cursor_ops:
c.set_key(1)
c.set_value(1)
if op == 'insert':
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c.insert(), msg)
elif op == 'remove':
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c.remove(), msg)
elif op == 'update':
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c.update(), msg)
else:
self.fail('Unknown cursor operation: ' + op)
c.close()
for op in self.session_ops:
if op == 'alter':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.alter(self.uri, None), msg)
elif op == 'create':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create(self.uri2, create_params), msg)
elif op == 'compact':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.compact(self.uri, None), msg)
elif op == 'drop':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.drop(self.uri, None), msg)
elif op == 'log_flush':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.log_flush(None), msg)
elif op == 'log_printf':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.log_printf("test"), msg)
elif op == 'rebalance':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.rebalance(self.uri, None), msg)
elif op == 'rename':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.rename(self.uri, self.uri2, None),
msg)
elif op == 'salvage':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.salvage(self.uri, None), msg)
elif op == 'truncate':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.truncate(self.uri, None, None, None),
msg)
elif op == 'upgrade':
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.upgrade(self.uri, None), msg)
else:
self.fail('Unknown session method: ' + op)
def test_checkpoint_target(self):
# Create 3 objects, change one record to an easily recognizable string.
uri = self.uri + '1'
ds1 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
ds1.populate()
self.update(uri, ds1, 'ORIGINAL')
uri = self.uri + '2'
ds2 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
ds2.populate()
self.update(uri, ds2, 'ORIGINAL')
uri = self.uri + '3'
ds3 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
ds3.populate()
self.update(uri, ds3, 'ORIGINAL')
# Checkpoint all three objects.
self.session.checkpoint("name=checkpoint-1")
# Update all 3 objects, then checkpoint two of the objects with the
# same checkpoint name.
self.update(self.uri + '1', ds1, 'UPDATE')
self.update(self.uri + '2', ds2, 'UPDATE')
self.update(self.uri + '3', ds3, 'UPDATE')
target = 'target=("' + self.uri + '1"' + ',"' + self.uri + '2")'
self.session.checkpoint("name=checkpoint-1," + target)
# Confirm the checkpoint has the old value in objects that weren't
# checkpointed, and the new value in objects that were checkpointed.
self.check(self.uri + '1', ds1, 'UPDATE')
self.check(self.uri + '2', ds2, 'UPDATE')
self.check(self.uri + '3', ds3, 'ORIGINAL')
def test_modify_abort(self):
ds = SimpleDataSet(self,
self.uri,
20,
key_format=self.keyfmt,
value_format=self.valuefmt)
ds.populate()
# Start a transaction.
self.session.begin_transaction()
# Insert a new record.
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(30))
c.set_value(ds.value(30))
self.assertEquals(c.insert(), 0)
# Test that we can successfully modify our own record.
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), 0)
# Test that another transaction cannot modify our uncommitted record.
xs = self.conn.open_session()
xc = xs.open_cursor(self.uri, None)
xs.begin_transaction()
xc.set_key(ds.key(30))
xc.set_value(ds.value(30))
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
xc.set_key(ds.key(30))
self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND)
xs.rollback_transaction()
# Rollback our transaction.
self.session.rollback_transaction()
# Test that we can't modify our aborted insert.
self.session.begin_transaction()
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
self.session.rollback_transaction()
def test_truncate07(self):
nrows = 10000
uri = "table:truncate07"
ds = SimpleDataSet(self,
uri,
0,
key_format=self.key_format,
value_format=self.value_format,
config=self.extraconfig)
ds.populate()
if self.value_format == '8t':
value_a = 97
value_b = 98
else:
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
# Pin oldest and stable timestamps to 1.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) +
',stable_timestamp=' + self.timestamp_str(1))
# Write a bunch of data at time 10.
cursor = self.session.open_cursor(ds.uri)
self.session.begin_transaction()
for i in range(1, nrows + 1):
cursor[ds.key(i)] = value_a
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(10))
# This data can be stable; move the stable timestamp forward.
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10))
# Munge some of it at time 20. Touch every other even-numbered key in the middle third of
# the data. (This allows using the odd keys to evict.)
#
# Use a separate session because we're going to prepare the transaction and we want to
# be able to do other things afterward.
session2 = self.conn.open_session()
cursor2 = session2.open_cursor(ds.uri)
session2.begin_transaction()
start = nrows // 3
if start % 2 == 1:
start += 1
for i in range(start, 2 * nrows // 3, 2):
cursor2.set_key(ds.key(i))
if self.munge_with_update:
cursor2.set_value(value_b)
self.assertEqual(cursor2.update(), 0)
else:
self.assertEqual(cursor2.remove(), 0)
session2.prepare_transaction('prepare_timestamp=' +
self.timestamp_str(20))
cursor2.close()
# Evict the lot so that we can fast-truncate.
# For now, evict every 4th key explicitly; FUTURE: improve this to evict each page only
# once when we have a good way to do that.
if self.do_evict:
for i in range(1, nrows + 1, 4):
self.evict(ds.uri, ds.key(i), value_a)
if self.do_checkpoint:
self.session.checkpoint()
# Truncate the data, including what we prepared.
self.session.begin_transaction()
err = self.truncate(ds.uri, ds.key, nrows // 4, nrows - nrows // 4)
self.assertEqual(err, WT_ROLLBACK)
self.session.rollback_transaction()
# Move the stable timestamp forward before exiting so we don't waste time rolling
# back the rest of the changes during shutdown.
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(50))
def test_rollback_to_stable(self):
nrows = 1000
# Prepare transactions for column store table is not yet supported.
if self.key_format == 'r':
self.skipTest(
'Prepare transactions for column store table is not yet supported'
)
# Create a table without logging.
uri = "table:rollback_to_stable21"
ds = SimpleDataSet(self,
uri,
0,
key_format=self.key_format,
value_format="S",
config='log=(enabled=false)')
ds.populate()
# Pin oldest and stable timestamps to 10.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) +
',stable_timestamp=' + timestamp_str(10))
valuea = 'a' * 400
valueb = 'b' * 400
cursor = self.session.open_cursor(uri)
self.session.begin_transaction()
for i in range(1, nrows + 1):
cursor[i] = valuea
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(30))
self.session.begin_transaction()
for i in range(1, nrows + 1):
cursor[i] = valueb
cursor.reset()
cursor.close()
self.session.prepare_transaction('prepare_timestamp=' +
timestamp_str(20))
s = self.conn.open_session()
s.begin_transaction('ignore_prepare = true')
# Configure debug behavior on a cursor to evict the page positioned on when the reset API is used.
evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)")
for i in range(1, nrows + 1):
evict_cursor.set_key(i)
self.assertEquals(evict_cursor.search(), 0)
self.assertEqual(evict_cursor.get_value(), valuea)
evict_cursor.reset()
s.rollback_transaction()
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40))
s.checkpoint()
# Rollback the prepared transaction
self.session.rollback_transaction()
# Simulate a server crash and restart.
self.pr("restart")
simulate_crash_restart(self, ".", "RESTART")
self.pr("restart complete")
self.check(valuea, uri, nrows, 40)
stat_cursor = self.session.open_cursor('statistics:', None, None)
hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2]
stat_cursor.close()
self.assertGreater(hs_removed, 0)
def test_timestamp_randomizer(self):
# Local function to generate a random timestamp, or return -1
def maybe_ts(do_gen, iternum):
if do_gen:
return self.gen_ts(iternum)
else:
return -1
if wttest.islongtest():
iterations = 100000
else:
iterations = 1000
create_params = 'key_format={},value_format={}'.format(self.key_format, self.value_format)
self.session.create(self.uri, create_params)
self.set_global_timestamps(1, 1, -1)
# Create tables with no entries
ds = SimpleDataSet(
self, self.uri, 0, key_format=self.key_format, value_format=self.value_format)
# We do a bunch of iterations, doing transactions, prepare, and global timestamp calls
# with timestamps that are sometimes valid, sometimes not. We use the iteration number
# as an "approximate timestamp", and generate timestamps for our calls that are near
# that number (within 10). Thus, as the test runs, the timestamps generally get larger.
# We always know the state of global timestamps, so we can predict the success/failure
# on each call.
self.commit_value = '<NOT_SET>'
for iternum in range(1, iterations):
self.pr('\n===== ITERATION ' + str(iternum) + '/' + str(iterations))
self.pr('RANDOM: ({0},{1})'.format(self.rand.seedw,self.rand.seedz))
if self.rand.rand32() % 10 != 0:
commit_ts = self.gen_ts(iternum)
durable_ts = self.gen_ts(iternum)
do_prepare = (self.rand.rand32() % 20 == 0)
if self.rand.rand32() % 2 == 0:
read_ts = self.gen_ts(iternum)
else:
read_ts = -1 # no read_timestamp used in txn
# OOD does not work with prepared updates. Hence, the commit ts should always be
# greater than the last durable ts.
if commit_ts <= self.last_durable:
commit_ts = self.last_durable + 1
if do_prepare:
# If we doing a prepare, we must abide by some additional rules.
# If we don't we'll immediately panic
if commit_ts < self.oldest_ts:
commit_ts = self.oldest_ts
if durable_ts < commit_ts:
durable_ts = commit_ts
if durable_ts <= self.stable_ts:
durable_ts = self.stable_ts + 1
value = self.gen_value(iternum, commit_ts)
self.updates(value, ds, do_prepare, commit_ts, durable_ts, read_ts)
if self.rand.rand32() % 2 == 0:
# Set some combination of the global timestamps
r = self.rand.rand32() % 16
oldest = maybe_ts((r & 0x1) != 0, iternum)
stable = maybe_ts((r & 0x2) != 0, iternum)
commit = maybe_ts((r & 0x4) != 0, iternum)
durable = maybe_ts((r & 0x8) != 0, iternum)
self.set_global_timestamps(oldest, stable, durable)
# Make sure the resulting rows are what we expect.
cursor = self.session.open_cursor(self.uri)
expect_key = 1
expect_value = self.commit_value
for k,v in cursor:
self.assertEquals(k, expect_key)
self.assertEquals(v, expect_value)
expect_key += 1
# Although it's theoretically possible to never successfully update a single row,
# with a large number of iterations that should never happen. I'd rather catch
# a test code error where we mistakenly don't update any rows.
self.assertGreater(expect_key, 1)
cursor.close()
def test_search_duplicate(self):
if self.colvar == 0:
return
# Populate the tree.
ds = SimpleDataSet(self,
self.uri,
105,
key_format=self.key_format,
value_format=self.value_format)
ds.populate()
# Set up deleted records before and after a set of duplicate records,
# and make sure search/search-near returns the correct record.
cursor = self.session.open_cursor(self.uri, None)
for i in range(20, 100):
cursor[ds.key(i)] = '=== IDENTICAL VALUE ==='
for i in range(15, 25):
cursor.set_key(ds.key(i))
self.assertEqual(cursor.remove(), 0)
for i in range(95, 106):
cursor.set_key(ds.key(i))
self.assertEqual(cursor.remove(), 0)
cursor.close()
# Reopen the connection, forcing it to disk and moving the records to
# an on-page format.
self.reopen_conn()
# Open a cursor.
cursor = self.session.open_cursor(self.uri, None)
# Search-near for a record in the deleted set before the duplicate set,
# which should succeed, returning the first record in the duplicate set.
cursor.set_key(ds.key(18))
self.assertEqual(cursor.search_near(), 1)
self.assertEqual(cursor.get_key(), ds.key(25))
# Search-near for a record in the deleted set after the duplicate set,
# which should succeed, returning the last record in the duplicate set.
cursor.set_key(ds.key(98))
self.assertEqual(cursor.search_near(), -1)
self.assertEqual(cursor.get_key(), ds.key(94))
def test_checkpoint_snapshot(self):
ds = SimpleDataSet(self, self.uri, 0, \
key_format=self.key_format, value_format=self.value_format, \
config='leaf_page_max=4k')
ds.populate()
if self.value_format == '8t':
valuea = 97
valueb = 98
valuec = 99
else:
valuea = "aaaaa" * 100
valueb = "bbbbb" * 100
valuec = "ccccc" * 100
session1 = self.conn.open_session()
session1.begin_transaction()
cursor1 = session1.open_cursor(self.uri)
for i in range(self.nrows + 1, self.nrows + 2):
cursor1.set_key(ds.key(i))
cursor1.set_value(valueb)
self.assertEqual(cursor1.insert(), 0)
self.large_updates(self.uri, valuea, ds, self.nrows)
self.check(valuea, self.uri, self.nrows)
self.session.checkpoint()
session1.rollback_transaction()
self.reopen_conn()
# Check the table contains the last checkpointed value.
self.session.breakpoint()
self.check(valuea, self.uri, self.nrows)
session1 = self.conn.open_session()
session1.begin_transaction()
cursor1 = session1.open_cursor(self.uri)
for i in range(self.nrows + 1, self.nrows + 2):
cursor1.set_key(ds.key(i))
cursor1.set_value(valueb)
self.assertEqual(cursor1.insert(), 0)
self.session.begin_transaction()
cursor = self.session.open_cursor(self.uri)
for i in range(1, 2):
cursor.set_key(ds.key(i))
cursor.set_value(valuec)
self.assertEqual(cursor.update(), 0)
self.session.commit_transaction()
self.session.checkpoint()
session1.rollback_transaction()
# Simulate a server crash and restart.
simulate_crash_restart(self, ".", "RESTART")
stat_cursor = self.session.open_cursor('statistics:', None, None)
inconsistent_ckpt = stat_cursor[stat.conn.txn_rts_inconsistent_ckpt][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2]
pages_skipped = stat_cursor[stat.conn.txn_rts_tree_walk_skip_pages][2]
upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2]
stat_cursor.close()
self.assertGreater(inconsistent_ckpt, 0)
self.assertEqual(upd_aborted, 0)
self.assertGreaterEqual(keys_removed, 0)
self.assertEqual(keys_restored, 0)
self.assertGreater(pages_skipped, 0)
def test_search_invisible_one(self):
# Populate the tree.
ds = SimpleDataSet(self,
self.uri,
100,
key_format=self.key_format,
value_format=self.value_format)
ds.populate()
# Delete a range of records.
for i in range(5, 10):
cursor = self.session.open_cursor(self.uri, None)
cursor.set_key(ds.key(i))
self.assertEqual(cursor.remove(), 0)
# Reopen the connection, forcing it to disk and moving the records to
# an on-page format.
self.reopen_conn()
# Add updates to the existing records (in both the deleted an undeleted
# range), as well as some new records after the end. Put the updates in
# a separate transaction so they're invisible to another cursor.
self.session.begin_transaction()
cursor = self.session.open_cursor(self.uri, None)
for i in range(5, 10):
cursor[ds.key(i)] = ds.value(i + 1000)
for i in range(30, 40):
cursor[ds.key(i)] = ds.value(i + 1000)
for i in range(100, 140):
cursor[ds.key(i)] = ds.value(i + 1000)
# Open a separate session and cursor.
s = self.conn.open_session()
cursor = s.open_cursor(self.uri, None)
# Search for an existing record in the deleted range, should not find
# it.
for i in range(5, 10):
cursor.set_key(ds.key(i))
if self.empty:
# Fixed-length column-store rows always exist.
self.assertEqual(cursor.search(), 0)
self.assertEqual(cursor.get_key(), i)
self.assertEqual(cursor.get_value(), 0)
else:
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
# Search for an existing record in the updated range, should see the
# original value.
for i in range(30, 40):
cursor.set_key(ds.key(i))
self.assertEqual(cursor.search(), 0)
self.assertEqual(cursor.get_key(), ds.key(i))
# Search for a added record, should not find it.
for i in range(120, 130):
cursor.set_key(ds.key(i))
if self.empty:
# Invisible updates to fixed-length column-store objects are
# invisible to the reader, but the fact that they exist past
# the end of the initial records causes the instantiation of
# empty records: confirm successful return of an empty row.
self.assertEqual(cursor.search(), 0)
self.assertEqual(cursor.get_key(), i)
self.assertEqual(cursor.get_value(), 0)
else:
# Otherwise, we should not find any matching records.
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
# Search-near for an existing record in the deleted range, should find
# the next largest record. (This depends on the implementation behavior
# which currently includes a bias to prefix search.)
for i in range(5, 10):
cursor.set_key(ds.key(i))
if self.empty:
# Fixed-length column-store rows always exist.
self.assertEqual(cursor.search_near(), 0)
self.assertEqual(cursor.get_key(), i)
self.assertEqual(cursor.get_value(), 0)
else:
self.assertEqual(cursor.search_near(), 1)
self.assertEqual(cursor.get_key(), ds.key(10))
# Search-near for an existing record in the updated range, should see
# the original value.
for i in range(30, 40):
cursor.set_key(ds.key(i))
self.assertEqual(cursor.search_near(), 0)
self.assertEqual(cursor.get_key(), ds.key(i))
# Search-near for an added record, should find the previous largest
# record.
for i in range(120, 130):
cursor.set_key(ds.key(i))
if self.empty:
# Invisible updates to fixed-length column-store objects are
# invisible to the reader, but the fact that they exist past
# the end of the initial records causes the instantiation of
# empty records: confirm successful return of an empty row.
self.assertEqual(cursor.search_near(), 0)
self.assertEqual(cursor.get_key(), i)
self.assertEqual(cursor.get_value(), 0)
else:
self.assertEqual(cursor.search_near(), -1)
self.assertEqual(cursor.get_key(), ds.key(100))
def test_update_restore_evict_recovery(self):
uri = 'table:test_debug_mode10'
nrows = 10000
# Create our table.
ds = SimpleDataSet(self, uri, 0, key_format='i', value_format='S',config='log=(enabled=false)')
ds.populate()
value_a = 'a' * 500
value_b = 'b' * 500
value_c = 'c' * 500
value_d = 'd' * 500
# Perform several updates.
self.large_updates(uri, value_a, ds, nrows, False, 20)
self.large_updates(uri, value_b, ds, nrows, False, 30)
self.large_updates(uri, value_c, ds, nrows, False, 40)
# Verify data is visible and correct.
self.check(value_a, uri, nrows, 20)
self.check(value_b, uri, nrows, 30)
self.check(value_c, uri, nrows, 40)
# Pin the stable timestamp to 40. We will be validating the state of the data post-stable timestamp
# after we perform a recovery.
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40))
# Perform additional updates post-stable timestamp.
self.large_updates(uri, value_d, ds, nrows, False, 50)
self.large_updates(uri, value_a, ds, nrows, False, 60)
self.large_updates(uri, value_b, ds, nrows, False, 70)
# Verify additional updated data is visible and correct.
self.check(value_d, uri, nrows, 50)
self.check(value_a, uri, nrows, 60)
self.check(value_b, uri, nrows, 70)
# Checkpoint to ensure the data is flushed to disk.
self.session.checkpoint()
# Extract the most recent checkpoints write gen & run write gen. As we are still on a new DB connection,
# the run write gen should be 1 at this point, equal to the connection-wide base write gen.
# Since we checkpointed after a series of large writes/updates, the write gen of the pages should
# definitely be greater than 1.
checkpoint_write_gen, checkpoint_run_write_gen = self.parse_write_gen("file:test_debug_mode10.wt")
self.assertEqual(checkpoint_run_write_gen, 1)
self.assertGreater(checkpoint_write_gen, checkpoint_run_write_gen)
# Simulate a crash/restart, opening our new DB in recovery. As we open in recovery we want to additionally
# use the 'update_restore_evict' debug option to trigger update restore eviction.
self.conn_config = self.conn_config + self.conn_recon
simulate_crash_restart(self, ".", "RESTART")
# As we've created a new DB connection post-shutdown, the connection-wide
# base write gen should eventually initialise from the previous checkpoint's base 'write_gen' during the recovery process
# ('write_gen'+1). This should be reflected in the initialisation of the 'run_write_gen' field of the newest
# checkpoint post-recovery. As the recovery/rts process updates our pages, we'd also expect the latest checkpoint's
# 'write_gen' to again be greater than its 'run_write_gen'.
recovery_write_gen, recovery_run_write_gen = self.parse_write_gen("file:test_debug_mode10.wt")
self.assertGreater(recovery_run_write_gen, checkpoint_write_gen)
self.assertGreater(recovery_write_gen, recovery_run_write_gen)
# Read the statistics of pages that have been update restored (to check the mechanism was used).
stat_cursor = self.session.open_cursor('statistics:')
pages_update_restored = stat_cursor[stat.conn.cache_write_restore][2]
stat_cursor.close()
self.assertGreater(pages_update_restored, 0)
# Check that after recovery, we see the correct data with respect to our previous stable timestamp (40).
self.check(value_c, uri, nrows, 40)
self.check(value_c, uri, nrows, 50)
self.check(value_c, uri, nrows, 60)
self.check(value_c, uri, nrows, 70)
self.check(value_b, uri, nrows, 30)
self.check(value_a, uri, nrows, 20)
# Passing 0 results in opening a transaction with no read timestamp.
self.check(value_c, uri, nrows, 0)
def test_prepare_rollback_retrieve_time_window(self):
# Create a small table.
uri = "table:test_prepare10"
nrows = 1000
ds = SimpleDataSet(self, uri, 0, key_format="S", value_format='u')
ds.populate()
value_a = b"aaaaa" * 100
value_b = b"bbbbb" * 100
value_c = b"ccccc" * 100
# Commit some updates along with a prepared update, which is not resolved.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10))
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10))
# Initially load huge data
self.updates(ds, uri, nrows, value_a, 20)
# Add some more updates
self.updates(ds, uri, nrows, value_b, 30)
# Checkpoint
self.session.checkpoint()
# Validate that we do see the correct value.
session2 = self.setUpSessionOpen(self.conn)
cursor2 = session2.open_cursor(uri)
session2.begin_transaction()
for i in range(1, nrows):
cursor2.set_key(ds.key(i))
self.assertEquals(cursor2.search(), 0)
self.assertEquals(cursor2.get_value(), value_b)
session2.commit_transaction()
# Reset the cursor.
cursor2.reset()
session2.begin_transaction()
# Remove all keys
self.removes(ds, uri, nrows, 40)
# Validate that we do see the correct value.
session3 = self.setUpSessionOpen(self.conn)
cursor3 = session3.open_cursor(uri)
session3.begin_transaction()
for i in range(1, nrows):
cursor3.set_key(ds.key(i))
self.assertEquals(cursor3.search(), wiredtiger.WT_NOTFOUND)
session3.commit_transaction()
# Reset the cursor.
cursor3.reset()
session3.begin_transaction()
# Insert the updates from a prepare session and keep it open.
session_p = self.conn.open_session()
cursor_p = session_p.open_cursor(uri)
session_p.begin_transaction()
for i in range(1, nrows):
cursor_p.set_key(ds.key(i))
cursor_p.set_value(value_c)
self.assertEquals(cursor_p.insert(), 0)
session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(50))
self.check(ds, uri, nrows, value_a, 20)
self.check(ds, uri, nrows, value_b, 35)
self.check_not_found(ds, uri, nrows, 60)
#rollback the prepared session
session_p.rollback_transaction()
self.check(ds, uri, nrows, value_a, 20)
self.check(ds, uri, nrows, value_b, 35)
self.check_not_found(ds, uri, nrows, 60)
# session2 still can see the value_b
for i in range(1, nrows):
cursor2.set_key(ds.key(i))
self.assertEquals(cursor2.search(), 0)
self.assertEquals(cursor2.get_value(), value_b)
session2.commit_transaction()
# session3 still can't see a value
for i in range(1, nrows):
cursor3.set_key(ds.key(i))
self.assertEquals(cursor3.search(), wiredtiger.WT_NOTFOUND)
session3.commit_transaction()
# close sessions.
cursor_p.close()
session_p.close()
cursor2.close()
session2.close()
cursor3.close()
session3.close()
self.session.close()
def test_gc(self):
nrows = 10000
# Create a table without logging.
uri = "table:gc01"
ds = SimpleDataSet(self,
uri,
0,
key_format="i",
value_format="S",
config='log=(enabled=false)')
ds.populate()
# Pin oldest and stable to timestamp 1.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) +
',stable_timestamp=' + timestamp_str(1))
bigvalue = "aaaaa" * 100
bigvalue2 = "ddddd" * 100
self.large_updates(uri, bigvalue, ds, nrows, 10)
# Check that all updates are seen.
self.check(bigvalue, uri, nrows, 10)
self.large_updates(uri, bigvalue2, ds, nrows, 100)
# Check that the new updates are only seen after the update timestamp.
self.check(bigvalue2, uri, nrows, 100)
# Check that old updates are seen.
self.check(bigvalue, uri, nrows, 10)
# Pin oldest and stable to timestamp 100.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(100) +
',stable_timestamp=' + timestamp_str(100))
# Checkpoint to ensure that the history store is cleaned.
self.session.checkpoint()
self.check_gc_stats()
# Check that the new updates are only seen after the update timestamp.
self.check(bigvalue2, uri, nrows, 100)
# Load a slight modification with a later timestamp.
self.large_modifies(uri, 'A', ds, 10, 1, nrows, 110)
self.large_modifies(uri, 'B', ds, 20, 1, nrows, 120)
self.large_modifies(uri, 'C', ds, 30, 1, nrows, 130)
# Second set of update operations with increased timestamp.
self.large_updates(uri, bigvalue, ds, nrows, 200)
# Check that the new updates are only seen after the update timestamp.
self.check(bigvalue, uri, nrows, 200)
# Check that the modifies are seen.
bigvalue_modA = bigvalue2[0:10] + 'A' + bigvalue2[11:]
bigvalue_modB = bigvalue_modA[0:20] + 'B' + bigvalue_modA[21:]
bigvalue_modC = bigvalue_modB[0:30] + 'C' + bigvalue_modB[31:]
self.check(bigvalue_modA, uri, nrows, 110)
self.check(bigvalue_modB, uri, nrows, 120)
self.check(bigvalue_modC, uri, nrows, 130)
# Check that old updates are seen.
self.check(bigvalue2, uri, nrows, 100)
# Pin oldest and stable to timestamp 200.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(200) +
',stable_timestamp=' + timestamp_str(200))
# Checkpoint to ensure that the history store is cleaned.
self.session.checkpoint()
self.check_gc_stats()
# Check that the new updates are only seen after the update timestamp.
self.check(bigvalue, uri, nrows, 200)
# Load a slight modification with a later timestamp.
self.large_modifies(uri, 'A', ds, 10, 1, nrows, 210)
self.large_modifies(uri, 'B', ds, 20, 1, nrows, 220)
self.large_modifies(uri, 'C', ds, 30, 1, nrows, 230)
# Third set of update operations with increased timestamp.
self.large_updates(uri, bigvalue2, ds, nrows, 300)
# Check that the new updates are only seen after the update timestamp.
self.check(bigvalue2, uri, nrows, 300)
# Check that the modifies are seen.
bigvalue_modA = bigvalue[0:10] + 'A' + bigvalue[11:]
bigvalue_modB = bigvalue_modA[0:20] + 'B' + bigvalue_modA[21:]
bigvalue_modC = bigvalue_modB[0:30] + 'C' + bigvalue_modB[31:]
self.check(bigvalue_modA, uri, nrows, 210)
self.check(bigvalue_modB, uri, nrows, 220)
self.check(bigvalue_modC, uri, nrows, 230)
# Check that old updates are seen.
self.check(bigvalue, uri, nrows, 200)
# Pin oldest and stable to timestamp 300.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(300) +
',stable_timestamp=' + timestamp_str(300))
# Checkpoint to ensure that the history store is cleaned.
self.session.checkpoint()
self.check_gc_stats()
# Check that the new updates are only seen after the update timestamp.
self.check(bigvalue2, uri, nrows, 300)
def test_rollback_to_stable(self):
# RTS will fail if there are uncommitted prepared transactions, so skip tests of prepare
# with a runtime call to RTS, that doesn't add useful testing scenarios.
if self.prepare and not self.crash:
return
nrows = 10000
# Create a table without logging.
uri = "table:rollback_to_stable33"
ds = SimpleDataSet(
self, uri, 0, key_format=self.key_format, value_format=self.value_format,
config='log=(enabled=false)' + self.extraconfig)
ds.populate()
if self.value_format == '8t':
valuea = 97
valueb = 98
else:
valuea = "aaaaa" * 100
valueb = "bbbbb" * 100
# Pin oldest and stable timestamps to 10.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) +
',stable_timestamp=' + self.timestamp_str(10))
cursor = self.session.open_cursor(uri)
# Write some baseline data out at time 20.
self.session.begin_transaction()
for i in range(1, nrows + 1):
cursor[ds.key(i)] = self.mkdata(valuea, i)
# Make a new transaction every 97 keys so the transactions don't get huge.
if i % 97 == 0:
self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20))
self.session.begin_transaction()
self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(20))
# Write some more data out at time 30.
self.session.begin_transaction()
for i in range(1, nrows + 1):
cursor[ds.key(i)] = self.mkdata(valueb, i)
# Make a new transaction every 97 keys so the transactions don't get huge.
if i % 97 == 0:
self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30))
self.session.begin_transaction()
self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(30))
cursor.close()
# Evict the lot.
self.evict(ds, 1, nrows + 1, valueb)
# Move stable to 25 (after the baseline data).
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(25))
# Checkpoint.
self.session.checkpoint()
# Now fast-delete the lot at time 35.
# Use a separate session for this so that if we leave the truncate prepared it
# doesn't obstruct the rest of the test.
session2 = self.conn.open_session()
session2.begin_transaction()
lo_cursor = session2.open_cursor(uri)
lo_cursor.set_key(ds.key(nrows // 2 + 1))
hi_cursor = session2.open_cursor(uri)
hi_cursor.set_key(ds.key(nrows + 1))
session2.truncate(None, lo_cursor, hi_cursor, None)
if self.prepare:
session2.prepare_transaction('prepare_timestamp=' + self.timestamp_str(35))
else:
session2.commit_transaction('commit_timestamp=' + self.timestamp_str(35))
hi_cursor.close()
lo_cursor.close()
# Check stats to make sure we fast-deleted at least one page.
# Since VLCS and FLCS do not (yet) support fast-delete, instead assert we didn't.
stat_cursor = self.session.open_cursor('statistics:', None, None)
fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
if self.key_format == 'r':
self.assertEqual(fastdelete_pages, 0)
else:
self.assertGreater(fastdelete_pages, 0)
if self.second_checkpoint:
# Checkpoint again with the deletion.
self.session.checkpoint()
# Roll back, either via crashing or by explicit RTS.
if self.crash:
simulate_crash_restart(self, ".", "RESTART")
else:
self.conn.rollback_to_stable()
# We should see the original data at read-ts 20 and 30.
self.checkx(ds, nrows, 20, valuea)
self.checkx(ds, nrows, 30, valuea)
def test_gc(self):
uri = "table:gc05"
create_params = 'value_format=S,key_format=i'
self.session.create(uri, create_params)
nrows = 10000
value_x = "xxxxx" * 100
value_y = "yyyyy" * 100
value_z = "zzzzz" * 100
ds = SimpleDataSet(self,
uri,
0,
key_format="i",
value_format="S",
config='log=(enabled=false)')
ds.populate()
# Set the oldest and stable timestamps to 10.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) +
',stable_timestamp=' + timestamp_str(10))
# Insert values with varying timestamps.
self.large_updates(uri, value_x, ds, nrows, 20)
self.large_updates(uri, value_y, ds, nrows, 30)
self.large_updates(uri, value_z, ds, nrows, 40)
# Perform a checkpoint.
self.session.checkpoint("name=checkpoint_one")
# Check statistics.
self.check_gc_stats()
# Open a cursor to the checkpoint just performed.
ckpt_cursor = self.session.open_cursor(uri, None,
"checkpoint=checkpoint_one")
# Move the oldest and stable timestamps to 40.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(40) +
',stable_timestamp=' + timestamp_str(40))
# Insert values with varying timestamps.
self.large_updates(uri, value_z, ds, nrows, 50)
self.large_updates(uri, value_y, ds, nrows, 60)
self.large_updates(uri, value_x, ds, nrows, 70)
# Move the oldest and stable timestamps to 70.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(70) +
',stable_timestamp=' + timestamp_str(70))
# Perform a checkpoint.
self.session.checkpoint()
self.check_gc_stats()
# Verify checkpoint_one still exists and contains the expected values.
for i in range(0, nrows):
ckpt_cursor.set_key(i)
ckpt_cursor.search()
self.assertEqual(value_z, ckpt_cursor.get_value())
# Close checkpoint cursor.
ckpt_cursor.close()
def test_hs(self):
active_files = []
value1 = 'a' * 500
value2 = 'd' * 500
# Set up 'numfiles' with 'numrows' entries. We want to create a number of files that
# contain active history (content newer than the oldest timestamp).
for f in range(self.numfiles):
table_uri = 'table:%s.%d' % (self.file_name, f)
file_uri = 'file:%s.%d.wt' % (self.file_name, f)
# Create a small table.
ds = SimpleDataSet(
self, table_uri, 0, key_format='S', value_format='S', config='log=(enabled=false)')
ds.populate()
# Checkpoint to ensure we write the files metadata checkpoint value.
self.session.checkpoint()
# Get the base write gen of the file so we can compare after the handles get closed.
base_write_gen = self.parse_run_write_gen(file_uri)
active_files.append((base_write_gen, ds))
# Pin oldest and stable to timestamp 1.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) +
',stable_timestamp=' + timestamp_str(1))
# Perform a series of updates over our files at timestamp 2. This being data we can later assert
# to ensure the history store is working as intended.
for (_, ds) in active_files:
# Load data at timestamp 2.
self.large_updates(ds.uri, value1, ds, self.nrows // 2 , 2)
# We want to create a long running read transaction in a seperate session which we will persist over the closing and
# re-opening of handles. We want to ensure the correct data gets read throughout this time period.
session_read = self.conn.open_session()
session_read.begin_transaction('read_timestamp=' + timestamp_str(2))
# Check our inital set of updates are seen at the read timestamp.
for (_, ds) in active_files:
# Check that all updates at timestamp 2 are seen.
self.check(session_read, value1, ds.uri, self.nrows // 2)
# Perform a series of updates over over files at a later timestamp. Checking the history store data is consistent
# with old and new timestamps.
for (_, ds) in active_files:
# Load more data with a later timestamp.
self.large_updates(ds.uri, value2, ds, self.nrows, 100)
# Check that the new updates are only seen after the update timestamp.
self.check(self.session, value1, ds.uri, self.nrows // 2, 2)
self.check(self.session, value2, ds.uri, self.nrows, 100)
# Our sweep scan interval is every 1 second and the amount of idle time needed for a handle to be closed is 2 seconds.
# It should take roughly 3 seconds for the sweep server to close our file handles. Lets wait at least double
# that to be safe.
max = 6
sleep = 0
# After waiting for the sweep server to remove our idle handles, the only open
# handles that should be the metadata file, history store file and lock file.
final_numfiles = 3
# Open the stats cursor to collect the dhandle sweep status.
stat_cursor = self.session.open_cursor('statistics:', None, None)
while sleep < max:
# We continue doing checkpoints which as a side effect runs the session handle sweep. This encouraging the idle
# handles get removed.
# Note, though checkpointing blocks sweeping, the checkpoint should be fast and not add too much extra time to the
# overall test time.
self.session.checkpoint()
sleep += 0.5
time.sleep(0.5)
stat_cursor.reset()
curr_files_open = stat_cursor[stat.conn.file_open][2]
curr_dhandles_removed = stat_cursor[stat.conn.dh_sweep_remove][2]
curr_dhandle_sweep_closes = stat_cursor[stat.conn.dh_sweep_close][2]
self.printVerbose(3, "==== loop " + str(sleep))
self.printVerbose(3, "Number of files open: " + str(curr_files_open))
self.printVerbose(3, "Number of connection sweep dhandles closed: " + str(curr_dhandle_sweep_closes))
self.printVerbose(3, "Number of connection sweep dhandles removed from hashlist: " + str(curr_dhandles_removed))
# We've sweeped all the handles we can if we are left with the number of final dhandles
# that we expect to be always open.
if curr_files_open == final_numfiles and curr_dhandle_sweep_closes >= self.numfiles:
break
stat_cursor.reset()
final_dhandle_sweep_closes = stat_cursor[stat.conn.dh_sweep_close][2]
stat_cursor.close()
# We want to assert our active history files have all been closed.
self.assertGreaterEqual(final_dhandle_sweep_closes, self.numfiles)
# Using our long running read transaction, we want to now check the correct data can still be read after the
# handles have been closed.
for (_, ds) in active_files:
# Check that all updates at timestamp 2 are seen.
self.check(session_read, value1, ds.uri, self.nrows // 2)
session_read.rollback_transaction()
# Perform a series of checks over our files to ensure that our transactions have been written
# before the dhandles were closed/sweeped.
# Also despite the dhandle is being re-opened, we don't expect the base write generation
# to have changed since we haven't actually restarted the system.
for idx, (initial_base_write_gen, ds) in enumerate(active_files):
# Check that the most recent transaction has the correct data.
self.check(self.session, value2, ds.uri, self.nrows, 100)
file_uri = 'file:%s.%d.wt' % (self.file_name, idx)
# Get the current base_write_gen and ensure it hasn't changed since being
# closed.
base_write_gen = self.parse_run_write_gen(file_uri)
self.assertEqual(initial_base_write_gen, base_write_gen)
def test_rollback_to_stable(self):
nrows = 10000
# Prepare transactions for column store table is not yet supported.
if self.prepare and self.key_format == 'r':
self.skipTest(
'Prepare transactions for column store table is not yet supported'
)
# Create a table without logging.
uri = "table:rollback_to_stable02"
ds = SimpleDataSet(self,
uri,
0,
key_format=self.key_format,
value_format="S",
config='log=(enabled=false)')
ds.populate()
# Pin oldest and stable to timestamp 1.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) +
',stable_timestamp=' + timestamp_str(1))
valuea = "aaaaa" * 100
valueb = "bbbbb" * 100
valuec = "ccccc" * 100
valued = "ddddd" * 100
self.large_updates(uri, valuea, ds, nrows, self.prepare, 10)
# Check that all updates are seen.
self.check(valuea, uri, nrows, 10)
self.large_updates(uri, valueb, ds, nrows, self.prepare, 20)
# Check that the new updates are only seen after the update timestamp.
self.check(valueb, uri, nrows, 20)
self.large_updates(uri, valuec, ds, nrows, self.prepare, 30)
# Check that the new updates are only seen after the update timestamp.
self.check(valuec, uri, nrows, 30)
self.large_updates(uri, valued, ds, nrows, self.prepare, 40)
# Check that the new updates are only seen after the update timestamp.
self.check(valued, uri, nrows, 40)
# Pin stable to timestamp 30 if prepare otherwise 20.
if self.prepare:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(30))
else:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(20))
# Checkpoint to ensure that all the data is flushed.
if not self.in_memory:
self.session.checkpoint()
self.conn.rollback_to_stable()
# Check that the new updates are only seen after the update timestamp.
self.check(valueb, uri, nrows, 40)
self.check(valueb, uri, nrows, 20)
self.check(valuea, uri, nrows, 10)
stat_cursor = self.session.open_cursor('statistics:', None, None)
calls = stat_cursor[stat.conn.txn_rts][2]
upd_aborted = (stat_cursor[stat.conn.txn_rts_upd_aborted][2] +
stat_cursor[stat.conn.txn_rts_hs_removed][2])
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2]
pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2]
stat_cursor.close()
self.assertEqual(calls, 1)
self.assertEqual(keys_removed, 0)
self.assertEqual(keys_restored, 0)
self.assertGreater(pages_visited, 0)
self.assertGreaterEqual(upd_aborted, nrows * 2)
def test_sharing(self):
ds = SimpleDataSet(self, self.uri, 10)
ds.populate()
ds.check()
self.session.checkpoint()
ds.check()
# Create a secondary database
dir2 = os.path.join(self.home, 'SECONDARY')
os.mkdir(dir2)
conn2 = self.setUpConnectionOpen(dir2)
session2 = conn2.open_session()
# Reference the tree from the secondary:
metac = self.session.open_cursor('metadata:')
metac2 = session2.open_cursor('metadata:', None, 'readonly=0')
uri2 = self.uri[:5] + '../' + self.uri[5:]
metac2[uri2] = metac[self.uri] + ",readonly=1"
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
newds = SimpleDataSet(self, self.uri, 10000)
newds.populate()
newds.check()
self.session.checkpoint()
newds.check()
# Check we can still read from the last checkpoint
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
# Bump to new checkpoint
origmeta = metac[self.uri]
checkpoint = re.search(r',checkpoint=\(.+?\)\)', origmeta).group(0)[1:]
self.pr('Orig checkpoint: ' + checkpoint)
session2.alter(uri2, checkpoint)
self.pr('New metadata on secondaery: ' + metac2[uri2])
# Check that we can see the new data
cursor2 = session2.open_cursor(uri2)
newds.check_cursor(cursor2)
def test_rollback_to_stable(self):
nrows = 1000
# Create a table without logging.
uri = "table:rollback_to_stable07"
ds = SimpleDataSet(
self, uri, 0, key_format="i", value_format="S", config='log=(enabled=false)')
ds.populate()
# Pin oldest and stable to timestamp 10.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) +
',stable_timestamp=' + timestamp_str(10))
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
value_c = "ccccc" * 100
value_d = "ddddd" * 100
# Perform several updates.
self.large_updates(uri, value_d, ds, nrows, 20)
self.large_updates(uri, value_c, ds, nrows, 30)
self.large_updates(uri, value_b, ds, nrows, 40)
self.large_updates(uri, value_a, ds, nrows, 50)
# Verify data is visible and correct.
self.check(value_d, uri, nrows, 20)
self.check(value_c, uri, nrows, 30)
self.check(value_b, uri, nrows, 40)
self.check(value_a, uri, nrows, 50)
# Pin stable to timestamp 50 if prepare otherwise 40.
if self.prepare:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50))
else:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40))
# Perform additional updates.
self.large_updates(uri, value_b, ds, nrows, 60)
self.large_updates(uri, value_c, ds, nrows, 70)
self.large_updates(uri, value_d, ds, nrows, 80)
# Checkpoint to ensure the data is flushed to disk.
self.session.checkpoint()
# Verify additional update data is visible and correct.
self.check(value_b, uri, nrows, 60)
self.check(value_c, uri, nrows, 70)
self.check(value_d, uri, nrows, 80)
# Simulate a server crash and restart.
simulate_crash_restart(self, ".", "RESTART")
# Check that the correct data is seen at and after the stable timestamp.
self.check(value_b, uri, nrows, 40)
self.check(value_b, uri, nrows, 80)
self.check(value_c, uri, nrows, 30)
self.check(value_d, uri, nrows, 20)
stat_cursor = self.session.open_cursor('statistics:', None, None)
calls = stat_cursor[stat.conn.txn_rts][2]
hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2]
pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2]
upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2]
stat_cursor.close()
self.assertEqual(calls, 0)
self.assertEqual(keys_removed, 0)
self.assertEqual(keys_restored, 0)
self.assertGreaterEqual(upd_aborted, 0)
self.assertGreater(pages_visited, 0)
self.assertGreaterEqual(hs_removed, nrows * 4)
# Simulate another server crash and restart.
simulate_crash_restart(self, "RESTART", "RESTART2")
# Check that the correct data is seen at and after the stable timestamp.
self.check(value_b, uri, nrows, 40)
self.check(value_b, uri, nrows, 80)
self.check(value_c, uri, nrows, 30)
self.check(value_d, uri, nrows, 20)
stat_cursor = self.session.open_cursor('statistics:', None, None)
calls = stat_cursor[stat.conn.txn_rts][2]
hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2]
pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2]
upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2]
stat_cursor.close()
self.assertEqual(calls, 0)
self.assertEqual(keys_removed, 0)
self.assertEqual(keys_restored, 0)
self.assertGreaterEqual(pages_visited, 0)
self.assertEqual(upd_aborted, 0)
self.assertEqual(hs_removed, 0)
def test_rollback_to_stable(self):
nrows = 1000
# Create a table.
uri = "table:rollback_to_stable13"
ds = SimpleDataSet(self,
uri,
0,
key_format=self.key_format,
value_format=self.value_format,
config='split_pct=50')
ds.populate()
if self.value_format == '8t':
value_a = 97
value_b = 98
else:
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
# Pin oldest and stable to timestamp 10.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) +
',stable_timestamp=' + self.timestamp_str(10))
# Perform several updates.
self.large_updates(uri, value_a, ds, nrows, self.prepare, 20)
# Perform several removes.
self.large_removes(uri, ds, nrows, self.prepare, 30)
# Perform several updates.
self.large_updates(uri, value_b, ds, nrows, self.prepare, 60)
# Verify data is visible and correct.
# (In FLCS, the removed rows should read back as zero.)
self.check(value_a, uri, nrows, None, 20)
self.check(None, uri, 0, nrows, 30)
self.check(value_b, uri, nrows, None, 60)
# Pin stable to timestamp 50 if prepare otherwise 40.
if self.prepare:
self.conn.set_timestamp('stable_timestamp=' +
self.timestamp_str(50))
else:
self.conn.set_timestamp('stable_timestamp=' +
self.timestamp_str(40))
self.session.checkpoint()
# Simulate a server crash and restart.
simulate_crash_restart(self, ".", "RESTART")
# Check that the correct data is seen at and after the stable timestamp.
self.check(None, uri, 0, nrows, 50)
# Check that we restore the correct value from the history store.
self.check(value_a, uri, nrows, None, 20)
stat_cursor = self.session.open_cursor('statistics:', None, None)
restored_tombstones = stat_cursor[
stat.conn.txn_rts_hs_restore_tombstones][2]
self.assertEqual(restored_tombstones, nrows)
def test_rollback_to_stable(self):
nrows = 1000
# Prepare transactions for column store table is not yet supported.
if self.prepare and self.key_format == 'r':
self.skipTest(
'Prepare transactions for column store table is not yet supported'
)
# Create a table without logging.
self.pr("create/populate tables")
uri_1 = "table:rollback_to_stable10_1"
ds_1 = SimpleDataSet(self,
uri_1,
0,
key_format=self.key_format,
value_format="S",
config='log=(enabled=false)')
ds_1.populate()
# Create another table without logging.
uri_2 = "table:rollback_to_stable10_2"
ds_2 = SimpleDataSet(self,
uri_2,
0,
key_format="i",
value_format="S",
config='log=(enabled=false)')
ds_2.populate()
# Pin oldest and stable to timestamp 10.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) +
',stable_timestamp=' + timestamp_str(10))
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
value_c = "ccccc" * 100
value_d = "ddddd" * 100
value_e = "eeeee" * 100
value_f = "fffff" * 100
# Perform several updates.
self.pr("large updates")
self.large_updates(uri_1, value_d, ds_1, nrows, self.prepare, 20)
self.large_updates(uri_1, value_c, ds_1, nrows, self.prepare, 30)
self.large_updates(uri_1, value_b, ds_1, nrows, self.prepare, 40)
self.large_updates(uri_1, value_a, ds_1, nrows, self.prepare, 50)
self.large_updates(uri_2, value_d, ds_2, nrows, self.prepare, 20)
self.large_updates(uri_2, value_c, ds_2, nrows, self.prepare, 30)
self.large_updates(uri_2, value_b, ds_2, nrows, self.prepare, 40)
self.large_updates(uri_2, value_a, ds_2, nrows, self.prepare, 50)
# Verify data is visible and correct.
self.check(value_d, uri_1, nrows, 20)
self.check(value_c, uri_1, nrows, 30)
self.check(value_b, uri_1, nrows, 40)
self.check(value_a, uri_1, nrows, 50)
self.check(value_d, uri_2, nrows, 20)
self.check(value_c, uri_2, nrows, 30)
self.check(value_b, uri_2, nrows, 40)
self.check(value_a, uri_2, nrows, 50)
# Pin stable to timestamp 60 if prepare otherwise 50.
if self.prepare:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60))
else:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50))
# Create a checkpoint thread
done = threading.Event()
ckpt = checkpoint_thread(self.conn, done)
try:
self.pr("start checkpoint")
ckpt.start()
# Perform several updates in parallel with checkpoint.
# Rollbacks may occur when checkpoint is running, so retry as needed.
self.pr("updates")
retry_rollback(
self, 'update ds1, e', None, lambda: self.large_updates(
uri_1, value_e, ds_1, nrows, self.prepare, 70))
retry_rollback(
self, 'update ds2, e', None, lambda: self.large_updates(
uri_2, value_e, ds_2, nrows, self.prepare, 70))
retry_rollback(
self, 'update ds1, f', None, lambda: self.large_updates(
uri_1, value_f, ds_1, nrows, self.prepare, 80))
retry_rollback(
self, 'update ds2, f', None, lambda: self.large_updates(
uri_2, value_f, ds_2, nrows, self.prepare, 80))
finally:
done.set()
ckpt.join()
# Simulate a server crash and restart.
self.pr("restart")
simulate_crash_restart(self, ".", "RESTART")
self.pr("restart complete")
# Check that the correct data is seen at and after the stable timestamp.
self.check(value_a, uri_1, nrows, 50)
self.check(value_a, uri_1, nrows, 80)
self.check(value_b, uri_1, nrows, 40)
self.check(value_c, uri_1, nrows, 30)
self.check(value_d, uri_1, nrows, 20)
# Check that the correct data is seen at and after the stable timestamp.
self.check(value_c, uri_2, nrows, 30)
self.check(value_a, uri_2, nrows, 50)
self.check(value_a, uri_2, nrows, 80)
self.check(value_b, uri_2, nrows, 40)
self.check(value_d, uri_2, nrows, 20)
stat_cursor = self.session.open_cursor('statistics:', None, None)
calls = stat_cursor[stat.conn.txn_rts][2]
hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2]
hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2]
pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2]
upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2]
stat_cursor.close()
self.assertEqual(calls, 0)
self.assertEqual(keys_removed, 0)
self.assertEqual(keys_restored, 0)
self.assertGreaterEqual(upd_aborted, 0)
self.assertGreater(pages_visited, 0)
self.assertGreaterEqual(hs_removed, 0)
self.assertGreater(hs_sweep, 0)
def test_rollback_to_stable(self):
nrows = 1000
# Create a table.
uri = "table:rollback_to_stable06"
ds_config = ',log=(enabled=false)' if self.in_memory else ''
ds = SimpleDataSet(self, uri, 0,
key_format=self.key_format, value_format=self.value_format, config=ds_config)
ds.populate()
if self.value_format == '8t':
value_a = 97
value_b = 98
value_c = 99
value_d = 100
else:
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
value_c = "ccccc" * 100
value_d = "ddddd" * 100
# Pin oldest and stable to timestamp 10.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) +
',stable_timestamp=' + self.timestamp_str(10))
# Perform several updates.
self.large_updates(uri, value_a, ds, nrows, self.prepare, 20)
self.large_updates(uri, value_b, ds, nrows, self.prepare, 30)
self.large_updates(uri, value_c, ds, nrows, self.prepare, 40)
self.large_updates(uri, value_d, ds, nrows, self.prepare, 50)
# Verify data is visible and correct.
self.check(value_a, uri, nrows, None, 20)
self.check(value_b, uri, nrows, None, 30)
self.check(value_c, uri, nrows, None, 40)
self.check(value_d, uri, nrows, None, 50)
# Checkpoint to ensure the data is flushed, then rollback to the stable timestamp.
if not self.in_memory:
self.session.checkpoint()
self.conn.rollback_to_stable()
# Check that all keys are removed.
# (For FLCS, at least for now, they will read back as 0, meaning deleted, rather
# than disappear.)
self.check(value_a, uri, 0, nrows, 20)
self.check(value_b, uri, 0, nrows, 30)
self.check(value_c, uri, 0, nrows, 40)
self.check(value_d, uri, 0, nrows, 50)
stat_cursor = self.session.open_cursor('statistics:', None, None)
calls = stat_cursor[stat.conn.txn_rts][2]
hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2]
pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2]
upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2]
stat_cursor.close()
self.assertEqual(calls, 1)
self.assertEqual(keys_restored, 0)
self.assertGreater(pages_visited, 0)
self.assertGreaterEqual(keys_removed, 0)
if self.in_memory:
self.assertEqual(upd_aborted, nrows * 4)
self.assertEqual(hs_removed, 0)
else:
self.assertGreaterEqual(upd_aborted + hs_removed + keys_removed, nrows * 4)
def test_rollback_to_stable_prepare(self):
nrows = 1000
# Create a table without logging.
self.pr("create/populate tables")
uri_1 = "table:rollback_to_stable10_1"
ds_1 = SimpleDataSet(self,
uri_1,
0,
key_format="i",
value_format="S",
config='log=(enabled=false)')
ds_1.populate()
# Create another table without logging.
uri_2 = "table:rollback_to_stable10_2"
ds_2 = SimpleDataSet(self,
uri_2,
0,
key_format="i",
value_format="S",
config='log=(enabled=false)')
ds_2.populate()
# Pin oldest and stable to timestamp 10.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) +
',stable_timestamp=' + timestamp_str(10))
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
value_c = "ccccc" * 100
value_d = "ddddd" * 100
value_e = "eeeee" * 100
value_f = "fffff" * 100
# Perform several updates.
self.pr("large updates")
self.large_updates(uri_1, value_d, ds_1, nrows, self.prepare, 20)
self.large_updates(uri_1, value_c, ds_1, nrows, self.prepare, 30)
self.large_updates(uri_1, value_b, ds_1, nrows, self.prepare, 40)
self.large_updates(uri_1, value_a, ds_1, nrows, self.prepare, 50)
self.large_updates(uri_2, value_d, ds_2, nrows, self.prepare, 20)
self.large_updates(uri_2, value_c, ds_2, nrows, self.prepare, 30)
self.large_updates(uri_2, value_b, ds_2, nrows, self.prepare, 40)
self.large_updates(uri_2, value_a, ds_2, nrows, self.prepare, 50)
# Verify data is visible and correct.
self.check(value_d, uri_1, nrows, 20)
self.check(value_c, uri_1, nrows, 30)
self.session.breakpoint()
self.check(value_b, uri_1, nrows, 40)
self.check(value_a, uri_1, nrows, 50)
self.check(value_d, uri_2, nrows, 20)
self.check(value_c, uri_2, nrows, 30)
self.session.breakpoint()
self.check(value_b, uri_2, nrows, 40)
self.check(value_a, uri_2, nrows, 50)
# Pin stable to timestamp 60 if prepare otherwise 50.
if self.prepare:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(60))
else:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50))
# Here's the update operations we'll perform, encapsulated so we can easily retry
# it if we get a rollback. Rollbacks may occur when checkpoint is running.
def prepare_range_updates(session, cursor, ds, value, nrows,
prepare_config):
self.pr("updates")
for i in range(1, nrows):
key = ds.key(i)
cursor.set_key(key)
cursor.set_value(value)
self.assertEquals(cursor.update(), 0)
self.pr("prepare")
session.prepare_transaction(prepare_config)
# Create a checkpoint thread
done = threading.Event()
ckpt = checkpoint_thread(self.conn, done)
try:
self.pr("start checkpoint")
ckpt.start()
# Perform several updates in parallel with checkpoint.
session_p1 = self.conn.open_session()
cursor_p1 = session_p1.open_cursor(uri_1)
session_p1.begin_transaction('isolation=snapshot')
retry_rollback(
self, 'update ds1', session_p1, lambda: prepare_range_updates(
session_p1, cursor_p1, ds_1, value_e, nrows,
'prepare_timestamp=' + timestamp_str(69)))
# Perform several updates in parallel with checkpoint.
session_p2 = self.conn.open_session()
cursor_p2 = session_p2.open_cursor(uri_2)
session_p2.begin_transaction('isolation=snapshot')
retry_rollback(
self, 'update ds2', session_p2, lambda: prepare_range_updates(
session_p2, cursor_p2, ds_2, value_e, nrows,
'prepare_timestamp=' + timestamp_str(69)))
finally:
done.set()
ckpt.join()
# Check that the history store file has been used and has non-zero size before the simulated
# crash.
stat_cursor = self.session.open_cursor('statistics:', None, None)
cache_hs_ondisk = stat_cursor[stat.conn.cache_hs_ondisk][2]
stat_cursor.close()
self.assertGreater(cache_hs_ondisk, 0)
# Simulate a crash by copying to a new directory(RESTART).
copy_wiredtiger_home(self, ".", "RESTART")
# Commit the prepared transaction.
session_p1.commit_transaction('commit_timestamp=' + timestamp_str(70) +
',durable_timestamp=' +
timestamp_str(71))
session_p2.commit_transaction('commit_timestamp=' + timestamp_str(70) +
',durable_timestamp=' +
timestamp_str(71))
session_p1.close()
session_p2.close()
# Open the new directory.
self.pr("restart")
self.conn = self.setUpConnectionOpen("RESTART")
self.session = self.setUpSessionOpen(self.conn)
self.pr("restart complete")
# The history store file size should be greater than zero after the restart.
stat_cursor = self.session.open_cursor('statistics:', None, None)
cache_hs_ondisk = stat_cursor[stat.conn.cache_hs_ondisk][2]
stat_cursor.close()
self.assertGreater(cache_hs_ondisk, 0)
# Check that the correct data is seen at and after the stable timestamp.
self.check(value_a, uri_1, nrows, 50)
self.check(value_a, uri_1, nrows, 80)
self.check(value_b, uri_1, nrows, 40)
self.check(value_c, uri_1, nrows, 30)
self.check(value_d, uri_1, nrows, 20)
# Check that the correct data is seen at and after the stable timestamp.
self.check(value_a, uri_2, nrows, 50)
self.check(value_a, uri_2, nrows, 80)
self.check(value_b, uri_2, nrows, 40)
self.check(value_c, uri_2, nrows, 30)
self.check(value_d, uri_2, nrows, 20)
stat_cursor = self.session.open_cursor('statistics:', None, None)
calls = stat_cursor[stat.conn.txn_rts][2]
hs_removed = stat_cursor[stat.conn.txn_rts_hs_removed][2]
hs_sweep = stat_cursor[stat.conn.txn_rts_sweep_hs_keys][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
keys_restored = stat_cursor[stat.conn.txn_rts_keys_restored][2]
pages_visited = stat_cursor[stat.conn.txn_rts_pages_visited][2]
upd_aborted = stat_cursor[stat.conn.txn_rts_upd_aborted][2]
stat_cursor.close()
self.assertEqual(calls, 0)
self.assertEqual(keys_removed, 0)
self.assertEqual(keys_restored, 0)
self.assertGreaterEqual(upd_aborted, 0)
self.assertGreater(pages_visited, 0)
self.assertGreaterEqual(hs_removed, 0)
self.assertGreater(hs_sweep, 0)
# The test may output the following message in eviction under cache pressure. Ignore that.
self.ignoreStdoutPatternIfExists(
"oldest pinned transaction ID rolled back for eviction")
def test_tiered(self):
args = 'key_format=S,block_allocation=log-structured'
self.verbose(
3, 'Test log-structured allocation with config: ' + args +
' count: ' + str(self.nrecs))
#ds = SimpleDataSet(self, self.uri, self.nrecs, config=args)
ds = SimpleDataSet(self, self.uri, 10, config=args)
ds.populate()
self.session.checkpoint()
ds = SimpleDataSet(self, self.uri, 10000, config=args)
ds.populate()
self.reopen_conn()
ds = SimpleDataSet(self, self.uri, 1000, config=args)
ds.populate()
def test_checkpoint_snapshot_with_txnid_and_timestamp(self):
self.moresetup()
ds = SimpleDataSet(self, self.uri, 0, \
key_format=self.key_format, value_format=self.value_format, \
config='log=(enabled=false)'+self.extraconfig)
ds.populate()
# Pin oldest and stable timestamps to 10.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) +
',stable_timestamp=' + self.timestamp_str(10))
session1 = self.conn.open_session()
session1.begin_transaction()
self.large_updates(self.uri, self.valuea, ds, self.nrows, 20)
self.check(self.valuea, self.uri, self.nrows, 20, False)
session2 = self.conn.open_session()
session2.begin_transaction()
cursor2 = session2.open_cursor(self.uri)
for i in range((self.nrows + 1), (self.nrows * 2) + 1):
cursor2.set_key(ds.key(i))
cursor2.set_value(self.valuea)
self.assertEqual(cursor2.insert(), 0)
session1.timestamp_transaction('commit_timestamp=' +
self.timestamp_str(30))
# Set stable timestamp to 40
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40))
# Create a checkpoint thread
done = threading.Event()
ckpt = checkpoint_thread(self.conn, done)
try:
ckpt.start()
# Sleep for sometime so that checkpoint starts before committing last transaction.
time.sleep(2)
session2.commit_transaction()
finally:
done.set()
ckpt.join()
session1.rollback_transaction()
self.perform_backup_or_crash_restart(".", self.backup_dir)
# Check the table contains the last checkpointed value.
self.check(self.valuea, self.uri, self.nrows, 30, True)
stat_cursor = self.session.open_cursor('statistics:', None, None)
inconsistent_ckpt = stat_cursor[stat.conn.txn_rts_inconsistent_ckpt][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
stat_cursor.close()
self.assertGreater(inconsistent_ckpt, 0)
self.assertGreaterEqual(keys_removed, 0)
self.perform_backup_or_crash_restart(self.backup_dir, self.backup_dir2)
# Check the table contains the last checkpointed value.
self.check(self.valuea, self.uri, self.nrows, 30, True)
stat_cursor = self.session.open_cursor('statistics:', None, None)
inconsistent_ckpt = stat_cursor[stat.conn.txn_rts_inconsistent_ckpt][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
stat_cursor.close()
self.assertGreaterEqual(inconsistent_ckpt, 0)
self.assertEqual(keys_removed, 0)
def test_search_eot(self):
# Populate the tree and reopen the connection, forcing it to disk
# and moving the records to an on-page format.
ds = SimpleDataSet(self,
self.uri,
100,
key_format=self.key_format,
value_format=self.value_format)
ds.populate()
self.reopen_conn()
# Open a cursor.
cursor = self.session.open_cursor(self.uri, None)
# Search for a record at the end of the table, which should succeed.
cursor.set_key(ds.key(100))
self.assertEqual(cursor.search(), 0)
self.assertEqual(cursor.get_key(), ds.key(100))
self.assertEqual(cursor.get_value(), ds.value(100))
# Search-near for a record at the end of the table, which should
# succeed, returning the last record.
cursor.set_key(ds.key(100))
self.assertEqual(cursor.search_near(), 0)
self.assertEqual(cursor.get_key(), ds.key(100))
self.assertEqual(cursor.get_value(), ds.value(100))
# Search for a record past the end of the table, which should fail.
cursor.set_key(ds.key(200))
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
# Search-near for a record past the end of the table, which should
# succeed, returning the last record.
cursor.set_key(ds.key(200))
self.assertEqual(cursor.search_near(), -1)
self.assertEqual(cursor.get_key(), ds.key(100))
self.assertEqual(cursor.get_value(), ds.value(100))
def test_search_invisible_two(self):
# Populate the tree and reopen the connection, forcing it to disk
# and moving the records to an on-page format.
ds = SimpleDataSet(self,
self.uri,
100,
key_format=self.key_format,
value_format=self.value_format)
ds.populate()
self.reopen_conn()
# Add some additional visible records.
cursor = self.session.open_cursor(self.uri, None)
for i in range(100, 120):
cursor[ds.key(i)] = ds.value(i)
cursor.close()
# Begin a transaction, and add some additional records.
self.session.begin_transaction()
cursor = self.session.open_cursor(self.uri, None)
for i in range(120, 140):
cursor[ds.key(i)] = ds.value(i)
# Open a separate session and cursor.
s = self.conn.open_session()
cursor = s.open_cursor(self.uri, None)
# Search for an invisible record.
cursor.set_key(ds.key(130))
if self.empty:
# Invisible updates to fixed-length column-store objects are
# invisible to the reader, but the fact that they exist past
# the end of the initial records causes the instantiation of
# empty records: confirm successful return of an empty row.
cursor.search()
self.assertEqual(cursor.get_key(), 130)
self.assertEqual(cursor.get_value(), 0)
else:
# Otherwise, we should not find any matching records.
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
# Search-near for an invisible record, which should succeed, returning
# the last visible record.
cursor.set_key(ds.key(130))
cursor.search_near()
if self.empty:
# Invisible updates to fixed-length column-store objects are
# invisible to the reader, but the fact that they exist past
# the end of the initial records causes the instantiation of
# empty records: confirm successful return of an empty row.
cursor.search()
self.assertEqual(cursor.get_key(), 130)
self.assertEqual(cursor.get_value(), 0)
else:
# Otherwise, we should find the closest record for which we can see
# the value.
self.assertEqual(cursor.get_key(), ds.key(119))
self.assertEqual(cursor.get_value(), ds.value(119))