def truncate(self, uri, make_key, keynum1, keynum2):
if self.trunc_with_remove:
cursor = self.session.open_cursor(uri)
err = 0
for k in range(keynum1, keynum2 + 1):
cursor.set_key(k)
try:
err = cursor.remove()
except WiredTigerError as e:
if wiredtiger_strerror(WT_ROLLBACK) in str(e):
err = WT_ROLLBACK
elif wiredtiger_strerror(WT_PREPARE_CONFLICT) in str(e):
err = WT_PREPARE_CONFLICT
else:
raise e
if err != 0:
break
cursor.close()
else:
lo_cursor = self.session.open_cursor(uri)
hi_cursor = self.session.open_cursor(uri)
lo_cursor.set_key(make_key(keynum1))
hi_cursor.set_key(make_key(keynum2))
try:
err = self.session.truncate(None, lo_cursor, hi_cursor, None)
except WiredTigerError as e:
if wiredtiger_strerror(WT_ROLLBACK) in str(e):
err = WT_ROLLBACK
elif wiredtiger_strerror(WT_PREPARE_CONFLICT) in str(e):
err = WT_PREPARE_CONFLICT
else:
raise e
lo_cursor.close()
hi_cursor.close()
return err
def retry_rollback(self, name, txn_session, code):
retry_limit = 100
retries = 0
completed = False
saved_exception = None
while not completed and retries < retry_limit:
if retries != 0:
self.pr("Retrying operation for " + name)
if txn_session:
txn_session.rollback_transaction()
sleep(0.1)
if txn_session:
txn_session.begin_transaction('isolation=snapshot')
self.pr("Began new transaction for " + name)
try:
code()
completed = True
except WiredTigerError as e:
rollback_str = wiredtiger_strerror(WT_ROLLBACK)
if rollback_str not in str(e):
raise(e)
retries += 1
saved_exception = e
if not completed and saved_exception:
raise(saved_exception)
def large_updates(self, uri, value, ds, nrows, prepare, commit_ts):
# Update a large number of records.
session = self.session
try:
cursor = session.open_cursor(uri)
for i in range(1, nrows + 1):
session.begin_transaction()
cursor[ds.key(i)] = value
if commit_ts == 0:
session.commit_transaction()
elif prepare:
session.prepare_transaction('prepare_timestamp=' +
self.timestamp_str(commit_ts -
1))
session.timestamp_transaction(
'commit_timestamp=' + self.timestamp_str(commit_ts))
session.timestamp_transaction(
'durable_timestamp=' +
self.timestamp_str(commit_ts + 1))
session.commit_transaction()
else:
session.commit_transaction('commit_timestamp=' +
self.timestamp_str(commit_ts))
cursor.close()
except WiredTigerError as e:
rollback_str = wiredtiger_strerror(WT_ROLLBACK)
if rollback_str in str(e):
session.rollback_transaction()
raise (e)
def large_modifies(self, uri, value, ds, location, nbytes, nrows, prepare, commit_ts):
# Load a slight modification.
session = self.session
try:
cursor = session.open_cursor(uri)
session.begin_transaction()
for i in range(1, nrows + 1):
cursor.set_key(i)
mods = [wiredtiger.Modify(value, location, nbytes)]
self.assertEqual(cursor.modify(mods), 0)
if commit_ts == 0:
session.commit_transaction()
elif prepare:
session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1))
session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts))
session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1))
session.commit_transaction()
else:
session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts))
cursor.close()
except WiredTigerError as e:
rollback_str = wiredtiger_strerror(WT_ROLLBACK)
if rollback_str in str(e):
session.rollback_transaction()
raise(e)
def prepare_call(self, func):
try:
ret = func()
except wiredtiger.WiredTigerError as e:
if wiredtiger.wiredtiger_strerror(wiredtiger.WT_PREPARE_CONFLICT) in str(e):
ret = wiredtiger.WT_PREPARE_CONFLICT
else:
raise e
return ret
def truncate(self, uri, make_key, keynum1, keynum2):
if self.trunc_with_remove:
cursor = self.session.open_cursor(uri)
err = 0
for k in range(keynum1, keynum2 + 1):
cursor.set_key(k)
try:
# In this test some or all the data is already deleted; skip those rows.
err = cursor.search()
if err == wiredtiger.WT_NOTFOUND:
err = 0
else:
err = cursor.remove()
except wiredtiger.WiredTigerError as e:
if wiredtiger.wiredtiger_strerror(
wiredtiger.WT_ROLLBACK) in str(e):
err = wiredtiger.WT_ROLLBACK
else:
raise e
if err != 0:
break
cursor.close()
else:
lo_cursor = self.session.open_cursor(uri)
hi_cursor = self.session.open_cursor(uri)
lo_cursor.set_key(make_key(keynum1))
hi_cursor.set_key(make_key(keynum2))
try:
err = self.session.truncate(None, lo_cursor, hi_cursor, None)
except wiredtiger.WiredTigerError as e:
if wiredtiger.wiredtiger_strerror(
wiredtiger.WT_ROLLBACK) in str(e):
err = wiredtiger.WT_ROLLBACK
else:
raise e
lo_cursor.close()
hi_cursor.close()
return err
def truncate(self, uri, key1, key2):
if self.trunc_with_remove:
# Because remove clears the cursor position, removing by cursor-next is a nuisance.
scan_cursor = self.session.open_cursor(uri)
del_cursor = self.session.open_cursor(uri)
err = 0
scan_cursor.set_key(key1)
self.assertEqual(scan_cursor.search(), 0)
while scan_cursor.get_key() <= key2:
del_cursor.set_key(scan_cursor.get_key())
try:
err = del_cursor.remove()
except WiredTigerError as e:
if wiredtiger_strerror(WT_ROLLBACK) in str(e):
err = WT_ROLLBACK
elif wiredtiger_strerror(WT_PREPARE_CONFLICT) in str(e):
err = WT_PREPARE_CONFLICT
else:
raise e
if err != 0:
break
if scan_cursor.get_key() == key2:
break
try:
err = scan_cursor.next()
except WiredTigerError as e:
if wiredtiger_strerror(WT_ROLLBACK) in str(e):
err = WT_ROLLBACK
elif wiredtiger_strerror(WT_PREPARE_CONFLICT) in str(e):
err = WT_PREPARE_CONFLICT
else:
raise e
if err != 0:
break
scan_cursor.close()
del_cursor.close()
else:
lo_cursor = self.session.open_cursor(uri)
hi_cursor = self.session.open_cursor(uri)
lo_cursor.set_key(key1)
hi_cursor.set_key(key2)
try:
err = self.session.truncate(None, lo_cursor, hi_cursor, None)
except WiredTigerError as e:
if wiredtiger_strerror(WT_ROLLBACK) in str(e):
err = WT_ROLLBACK
elif wiredtiger_strerror(WT_PREPARE_CONFLICT) in str(e):
err = WT_PREPARE_CONFLICT
else:
raise e
lo_cursor.close()
hi_cursor.close()
return err
def retry_rollback(self, name, code):
retry_limit = 100
retries = 0
completed = False
saved_exception = None
while not completed and retries < retry_limit:
if retries != 0:
self.pr("Retrying operation for " + name)
sleep(0.1)
try:
code()
completed = True
except WiredTigerError as e:
rollback_str = wiredtiger_strerror(WT_ROLLBACK)
if rollback_str not in str(e):
raise (e)
retries += 1
saved_exception = e
if not completed and saved_exception:
raise (saved_exception)
def large_modifies(self, uri, value, ds, location, nbytes, nrows, prepare,
commit_ts):
# Load a slight modification.
session = self.session
try:
cursor = session.open_cursor(uri)
session.begin_transaction()
for i in range(1, nrows + 1):
cursor.set_key(i)
# FLCS doesn't support modify (for obvious reasons) so just update.
# Use the first character of the passed-in value.
if self.value_format == '8t':
cursor.set_value(bytes(value, encoding='utf-8')[0])
self.assertEqual(cursor.update(), 0)
else:
mods = [wiredtiger.Modify(value, location, nbytes)]
self.assertEqual(cursor.modify(mods), 0)
if commit_ts == 0:
session.commit_transaction()
elif prepare:
session.prepare_transaction('prepare_timestamp=' +
self.timestamp_str(commit_ts - 1))
session.timestamp_transaction('commit_timestamp=' +
self.timestamp_str(commit_ts))
session.timestamp_transaction('durable_timestamp=' +
self.timestamp_str(commit_ts +
1))
session.commit_transaction()
else:
session.commit_transaction('commit_timestamp=' +
self.timestamp_str(commit_ts))
cursor.close()
except WiredTigerError as e:
rollback_str = wiredtiger_strerror(WT_ROLLBACK)
if rollback_str in str(e):
session.rollback_transaction()
raise (e)
def test_prepare(self):
self.create()
session2 = self.conn.open_session()
cursor = session2.open_cursor(self.uri, None)
# Add a value to the update chain
session2.begin_transaction()
cursor[1] = 0
session2.prepare_transaction("prepare_timestamp=" +
self.timestamp_str(1))
evict_cursor = self.session.open_cursor(self.uri, None,
"debug=(release_evict)")
self.session.begin_transaction()
evict_cursor.set_key(1)
try:
evict_cursor.search()
except wiredtiger.WiredTigerError as e:
if wiredtiger.wiredtiger_strerror(
wiredtiger.WT_PREPARE_CONFLICT) not in str(e):
raise e
evict_cursor.reset()
self.session.rollback_transaction()
# Open a version cursor
self.session.begin_transaction()
version_cursor = self.session.open_cursor(self.uri, None,
"debug=(dump_version=true)")
version_cursor.set_key(1)
self.assertEquals(version_cursor.search(), 0)
self.assertEquals(version_cursor.get_key(), 1)
self.verify_value(version_cursor, 1, 0, WT_TS_MAX, WT_TS_MAX, 3, 1, 4,
0, 0)
self.assertEquals(version_cursor.next(), 0)
self.assertEquals(version_cursor.get_key(), 1)
self.verify_value(version_cursor, 1, 1, 1, 0, 3, 1, 0, 1, 0)
self.assertEquals(version_cursor.next(), wiredtiger.WT_NOTFOUND)
def test_timestamp(self):
# Create a file that contains active history (content newer than the oldest timestamp).
table_uri = 'table:timestamp23'
ds = SimpleDataSet(self,
table_uri,
0,
key_format=self.key_format,
value_format='S',
config='log=(enabled=false)')
ds.populate()
self.session.checkpoint()
key = 5
value_1 = 'a' * 500
value_2 = 'b' * 500
value_3 = 'c' * 500
# Pin oldest and stable to timestamp 1.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) +
',stable_timestamp=' + self.timestamp_str(1))
cursor = self.session.open_cursor(ds.uri)
# Write two values at timestamp 10. We'll muck with the first value
# and use the second to reference the page for eviction.
self.session.begin_transaction('read_timestamp=10')
cursor[key] = value_1
cursor[key + 1] = value_2
self.session.commit_transaction('commit_timestamp=11')
# Delete the first value at timestamp 20.
self.session.begin_transaction('read_timestamp=20')
cursor.set_key(key)
cursor.remove()
self.session.commit_transaction('commit_timestamp=21')
# Put it back at timestamp 30.
self.session.begin_transaction('read_timestamp=30')
cursor[key] = value_3
self.session.commit_transaction('commit_timestamp=31')
# Delete it again at timestamp 40.
self.session.begin_transaction('read_timestamp=40')
cursor.set_key(key)
cursor.remove()
self.session.commit_transaction('commit_timestamp=41')
# Evict the page using the second key.
evict_cursor = self.session.open_cursor(ds.uri, None,
"debug=(release_evict)")
self.session.begin_transaction()
v = evict_cursor[key + 1]
self.assertEqual(v, value_2)
self.assertEqual(evict_cursor.reset(), 0)
self.session.rollback_transaction()
# Create a separate session and a cursor to read the original value at timestamp 12.
session2 = self.conn.open_session()
cursor2 = session2.open_cursor(ds.uri)
session2.begin_transaction('read_timestamp=12')
v = cursor2[key]
self.assertEqual(v, value_1)
self.session.breakpoint()
# Now delete the original value. This _should_ cause WT_ROLLBACK, but with a column
# store bug seen and fixed in August 2021, it succeeds, and the resulting invalid
# tombstone will cause reconciliation to assert. (To see this behavior, comment out the
# self.fail call and let the transaction commit.)
try:
cursor2.remove()
self.fail("Conflicting remove did not fail")
session2.commit_transaction('commit_timestamp=50')
except wiredtiger.WiredTigerError as e:
self.assertTrue(
wiredtiger.wiredtiger_strerror(wiredtiger.WT_ROLLBACK) in str(
e))
cursor.close()
cursor2.close()
def test_timestamp(self):
table_uri = 'table:timestamp24'
ds = SimpleDataSet(self,
table_uri,
0,
key_format=self.key_format,
value_format=self.value_format)
ds.populate()
self.session.checkpoint()
key = 5
if self.value_format == '8t':
value_a = 97
value_b = 98
value_c = 99
value_d = 100
else:
value_a = 'a' * 500
value_b = 'b' * 500
value_c = 'c' * 500
value_d = 'd' * 500
# Pin oldest and stable to timestamp 1.
#self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) +
# ',stable_timestamp=' + self.timestamp_str(1))
# Create two sessions so we can have two transactions.
session1 = self.session
session2 = self.conn.open_session()
# In session 1, write value_a at time 20.
# Commit that and then start a new transaction. Read the value back.
# Then reset the cursor so the page isn't pinned, but leave the transaction open.
cursor1 = session1.open_cursor(ds.uri)
session1.begin_transaction()
cursor1[key] = value_a
session1.commit_transaction('commit_timestamp=20')
session1.begin_transaction('read_timestamp=25')
tmp = cursor1[key]
self.assertEqual(tmp, value_a)
cursor1.reset()
# leave session1's transaction open
# In session 2, write value_b at time 50. Commit that.
cursor2 = session2.open_cursor(ds.uri)
session2.begin_transaction()
cursor2[key] = value_b
session2.commit_transaction('commit_timestamp=50')
cursor2.reset()
# Evict the page to force reconciliation. value_b goes to disk; value_a to history.
# Use session2 so we can keep session1's transaction open.
self.evict(ds.uri, session2, key, value_b)
# In session 2, write value_c, but abort it.
session2.begin_transaction()
cursor2[key] = value_c
session2.rollback_transaction()
# Now in session 1 try to write value_d. This should produce WT_ROLLBACK, but with
# a bug seen and fixed in August 2021, succeeds improperly instead, resulting in
# data corruption. The behavior is more exciting when the update is a modify (the
# modify gets applied to value_b instead of value_a, producing a more detectable
# corruption) but this is not necessary to check the wrong behavior.
try:
cursor1[key] = value_d
self.fail("Conflicting update did not fail")
broken = True
except wiredtiger.WiredTigerError as e:
self.assertTrue(
wiredtiger.wiredtiger_strerror(wiredtiger.WT_ROLLBACK) in str(
e))
broken = False
# Put this outside the try block in case it throws its own exceptions
if broken:
session1.commit_transaction('commit_timestamp=30')
else:
session1.rollback_transaction()
# Read the data back
session2.begin_transaction('read_timestamp=60')
tmp = cursor2[key]
# It should be value_b. But if we broke, it'll be value_d.
self.assertEqual(tmp, value_d if broken else value_b)
cursor2.close()
cursor1.close()
def test_truncate15(self):
# Note: 50000 is not large enough to trigger the problem.
nrows = 100000
uri = "table:truncate15"
ds = SimpleDataSet(self,
uri,
0,
key_format=self.key_format,
value_format=self.value_format,
config='log=(enabled=false)' + self.extraconfig)
ds.populate()
if self.value_format == '8t':
value_a = 97
value_b = 98
else:
value_a = "aaaaa" * 500
value_b = "bbbbb" * 500
# Pin oldest and stable timestamps to 1.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) +
',stable_timestamp=' + self.timestamp_str(1))
# Write a bunch of data at time 10.
cursor = self.session.open_cursor(ds.uri)
self.session.begin_transaction()
for i in range(1, nrows + 1):
cursor[ds.key(i)] = value_a
# Commit every 101 rows to avoid overflowing the cache.
if i % 101 == 0:
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(10))
self.session.begin_transaction()
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(10))
# Mark it stable.
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(10))
# Reopen the connection so nothing is in memory and we can fast-truncate.
self.reopen_conn()
# Truncate the data at time 25, but prepare at 20 and make durable 30.
self.session.begin_transaction()
err = self.truncate(ds.uri, ds.key, nrows // 4 + 1,
nrows // 4 + nrows // 2)
self.assertEqual(err, 0)
self.session.prepare_transaction('prepare_timestamp=' +
self.timestamp_str(20))
self.session.timestamp_transaction('commit_timestamp=' +
self.timestamp_str(25))
self.session.commit_transaction('durable_timestamp=' +
self.timestamp_str(30))
# Make sure we did at least one fast-delete. For FLCS, there's no fast-delete
# support, so assert we didn't.
stat_cursor = self.session.open_cursor('statistics:', None, None)
fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
if self.value_format == '8t':
self.assertEqual(fastdelete_pages, 0)
else:
self.assertGreater(fastdelete_pages, 0)
# Advance stable.
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30))
self.session.checkpoint()
# Reopen the connection so nothing is in memory.
self.reopen_conn(
".",
"cache_size=1MB,eviction_dirty_target=90,eviction_dirty_trigger=100"
+
",eviction_updates_target=90,eviction_updates_trigger=100,readonly=true"
)
# Validate the data.
try:
# At time 10 we should see all value_a.
self.check(ds.uri, ds.key, nrows, 0, value_a, 10)
#self.evict_cursor(ds.uri, ds, nrows, 10)
# At time 20 we should still see all value_a.
self.check(ds.uri, ds.key, nrows, 0, value_a, 20)
#self.evict_cursor(ds.uri, ds, nrows, 20)
# At time 25 we should still see half value_a, and for FLCS, half zeros.
self.check(ds.uri, ds.key, nrows // 2, nrows // 2, value_a, 25)
#self.evict_cursor(ds.uri, ds, nrows // 2, 25)
# At time 30 we should also see half value_a, and for FLCS, half zeros.
self.check(ds.uri, ds.key, nrows // 2, nrows // 2, value_a, 30)
#self.evict_cursor(ds.uri, ds, nrows // 2, 30)
except WiredTigerError as e:
# If we get WT_ROLLBACK while reading, assume it's because we overflowed the
# cache, and fail. (If we don't trap this explicitly, the test harness retries
# the whole test, which is not what we want.)
if wiredtiger_strerror(WT_ROLLBACK) in str(e):
self.assertTrue(False)
else:
raise e