def test_modify_smoke_recover(self):
# Close the original database.
self.conn.close()
# Open a new database with logging configured.
self.conn_config = \
'log=(enabled=true),transaction_sync=(method=dsync,enabled)'
self.conn = self.setUpConnectionOpen(".")
self.session = self.setUpSessionOpen(self.conn)
# Populate a database, and checkpoint it so it exists after recovery.
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.session.checkpoint()
self.modify_load(ds, False)
# Crash and recover in a new directory.
newdir = 'RESTART'
copy_wiredtiger_home('.', newdir)
self.conn.close()
self.conn = self.setUpConnectionOpen(newdir)
self.session = self.setUpSessionOpen(self.conn)
self.session.verify(self.uri)
self.modify_confirm(ds, False)
def test_prepare_lookaside(self):
if not wiredtiger.timestamp_build():
self.skipTest('requires a timestamp build')
# Create a small table.
uri = "table:test_prepare_lookaside01"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u')
ds.populate()
bigvalue = "aaaaa" * 100
# Initially load huge data
cursor = self.session.open_cursor(uri)
for i in range(1, 10000):
cursor.set_key(ds.key(nrows + i))
cursor.set_value(bigvalue)
self.assertEquals(cursor.insert(), 0)
cursor.close()
self.session.checkpoint()
# Check if lookaside is working properly with prepare transactions.
# We put prepared updates in multiple sessions so that we do not hang
# because of cache being full with uncommitted updates.
nsessions = 3
nkeys = 4000
self.prepare_updates(uri, ds, nrows, nsessions, nkeys)
def test_truncate_cursor_order(self):
uri = self.type + self.name
# A simple, one-file file or table object.
ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt)
ds.populate()
c1 = self.session.open_cursor(uri, None)
c1.set_key(ds.key(1000))
c2 = self.session.open_cursor(uri, None)
c2.set_key(ds.key(2000))
self.session.truncate(None, c1, c2, None)
self.assertEqual(c1.close(), 0)
self.assertEqual(c2.close(), 0)
self.session.drop(uri)
if self.type == "table:":
ds = ComplexDataSet(self, uri, 100, key_format=self.keyfmt)
ds.populate()
c1 = self.session.open_cursor(uri, None)
c1.set_key(ds.key(1000))
c2 = self.session.open_cursor(uri, None)
c2.set_key(ds.key(2000))
self.session.truncate(None, c1, c2, None)
self.assertEqual(c1.close(), 0)
self.assertEqual(c2.close(), 0)
self.session.drop(uri)
def test_search_eot(self):
# Populate the tree and reopen the connection, forcing it to disk
# and moving the records to an on-page format.
ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format,
value_format=self.value_format)
ds.populate()
self.reopen_conn()
# Open a cursor.
cursor = self.session.open_cursor(self.uri, None)
# Search for a record at the end of the table, which should succeed.
cursor.set_key(ds.key(100))
self.assertEqual(cursor.search(), 0)
self.assertEqual(cursor.get_key(), ds.key(100))
self.assertEqual(cursor.get_value(), ds.value(100))
# Search-near for a record at the end of the table, which should
# succeed, returning the last record.
cursor.set_key(ds.key(100))
self.assertEqual(cursor.search_near(), 0)
self.assertEqual(cursor.get_key(), ds.key(100))
self.assertEqual(cursor.get_value(), ds.value(100))
# Search for a record past the end of the table, which should fail.
cursor.set_key(ds.key(200))
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
# Search-near for a record past the end of the table, which should
# succeed, returning the last record.
cursor.set_key(ds.key(200))
self.assertEqual(cursor.search_near(), -1)
self.assertEqual(cursor.get_key(), ds.key(100))
self.assertEqual(cursor.get_value(), ds.value(100))
def test_drop(self):
uri = 'lsm:' + self.name
ds = SimpleDataSet(self, uri, 100000)
ds.populate()
self.reopen_conn()
self.session.drop(uri, None)
def test_basic(self):
ds = SimpleDataSet(self, self.uri, self.nentries,
config=self.config, key_format=self.keyfmt)
ds.populate()
self.reopen_conn()
c = self.session.open_cursor(self.uri, None)
self.forward(c, ds, self.nentries, [])
self.backward(c, ds, self.nentries, [])
def test_modify_smoke_single(self):
if self.skip():
return
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format='u')
ds.populate()
self.modify_load(ds, True)
def test_modify_smoke_reopen(self):
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.modify_load(ds, False)
# Flush to disk, forcing reconciliation.
self.reopen_conn()
self.modify_confirm(ds, False)
def test_search_invisible_two(self):
# Populate the tree and reopen the connection, forcing it to disk
# and moving the records to an on-page format.
ds = SimpleDataSet(self, self.uri, 100, key_format=self.key_format,
value_format=self.value_format)
ds.populate()
self.reopen_conn()
# Add some additional visible records.
cursor = self.session.open_cursor(self.uri, None)
for i in range(100, 120):
cursor[ds.key(i)] = ds.value(i)
cursor.close()
# Begin a transaction, and add some additional records.
self.session.begin_transaction()
cursor = self.session.open_cursor(self.uri, None)
for i in range(120, 140):
cursor[ds.key(i)] = ds.value(i)
# Open a separate session and cursor.
s = self.conn.open_session()
cursor = s.open_cursor(self.uri, None)
# Search for an invisible record.
cursor.set_key(ds.key(130))
if self.empty:
# Invisible updates to fixed-length column-store objects are
# invisible to the reader, but the fact that they exist past
# the end of the initial records causes the instantiation of
# empty records: confirm successful return of an empty row.
cursor.search()
self.assertEqual(cursor.get_key(), 130)
self.assertEqual(cursor.get_value(), 0)
else:
# Otherwise, we should not find any matching records.
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
# Search-near for an invisible record, which should succeed, returning
# the last visible record.
cursor.set_key(ds.key(130))
cursor.search_near()
if self.empty:
# Invisible updates to fixed-length column-store objects are
# invisible to the reader, but the fact that they exist past
# the end of the initial records causes the instantiation of
# empty records: confirm successful return of an empty row.
cursor.search()
self.assertEqual(cursor.get_key(), 130)
self.assertEqual(cursor.get_value(), 0)
else:
# Otherwise, we should find the closest record for which we can see
# the value.
self.assertEqual(cursor.get_key(), ds.key(119))
self.assertEqual(cursor.get_value(), ds.value(119))
def test_checkpoint_stats(self):
ds = SimpleDataSet(self, self.uri, self.nentries,
config=self.config, key_format=self.keyfmt)
for name in ('first', 'second', 'third'):
ds.populate()
self.session.checkpoint('name=' + name)
cursor = self.session.open_cursor(
'statistics:' + self.uri, None, 'checkpoint=' + name)
self.assertEqual(
cursor[stat.dsrc.btree_entries][2], self.nentries)
cursor.close()
def test_truncate_cursor_order(self):
uri = self.type + self.name
ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt)
ds.populate()
c1 = self.session.open_cursor(uri, None)
c2 = self.session.open_cursor(uri, None)
c1.set_key(ds.key(20))
c2.set_key(ds.key(10))
msg = "/the start cursor position is after the stop cursor position/"
self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg)
c2.set_key(ds.key(20))
self.session.truncate(None, c1, c2, None)
def test_las(self):
if not wiredtiger.timestamp_build():
self.skipTest('requires a timestamp build')
nrows = 10000
# Create a table without logging to ensure we get "skew_newest" lookaside eviction behavior.
uri = "table:las02_main"
ds = SimpleDataSet(
self, uri, 0, key_format="S", value_format="S", config='log=(enabled=false)')
ds.populate()
uri2 = "table:las02_extra"
ds2 = SimpleDataSet(self, uri2, 0, key_format="S", value_format="S")
ds2.populate()
# Pin oldest and stable to timestamp 1.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) +
',stable_timestamp=' + timestamp_str(1))
bigvalue = "aaaaa" * 100
self.large_updates(uri, bigvalue, ds, nrows / 3, 1)
# Check that all updates are seen
self.check(bigvalue, uri, nrows / 3, 1)
# Check to see lookaside working with old timestamp
bigvalue2 = "ddddd" * 100
self.large_updates(uri, bigvalue2, ds, nrows, 100)
# Check that the new updates are only seen after the update timestamp
self.check(bigvalue, uri, nrows / 3, 1)
self.check(bigvalue2, uri, nrows, 100)
# Force out most of the pages by updating a different tree
self.large_updates(uri2, bigvalue, ds2, nrows, 100)
# Now truncate half of the records
self.session.begin_transaction()
end = self.session.open_cursor(uri)
end.set_key(ds.key(nrows / 2))
self.session.truncate(None, None, end)
end.close()
self.session.commit_transaction('commit_timestamp=' + timestamp_str(200))
# Check that the truncate is visible after commit
self.check(bigvalue2, uri, nrows / 2, 200)
# Repeat earlier checks
self.check(bigvalue, uri, nrows / 3, 1)
self.check(bigvalue2, uri, nrows, 100)
def test_truncate_cursor_notset(self):
uri = self.type + self.name
msg = "/requires key be set/"
ds = SimpleDataSet(self, uri, 100)
ds.populate()
c1 = self.session.open_cursor(uri, None)
c2 = self.session.open_cursor(uri, None)
c2.set_key(ds.key(10))
self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg)
self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c2, c1, None), msg)
c1.close()
c2.close()
def test_las(self):
# Create a small table.
uri = "table:test_las"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format="S")
ds.populate()
bigvalue = "aaaaa" * 100
# Initially load huge data
cursor = self.session.open_cursor(uri)
for i in range(1, 10000):
cursor.set_key(ds.key(nrows + i))
cursor.set_value(bigvalue)
self.assertEquals(cursor.insert(), 0)
cursor.close()
self.session.checkpoint()
# Scenario: 1
# Check to see LAS working with old snapshot
bigvalue1 = "bbbbb" * 100
self.session.snapshot("name=xxx")
# Update the values in different session after snapshot
self.large_updates(self.session, uri, bigvalue1, ds, nrows)
# Check to see the value after recovery
self.durable_check(bigvalue1, uri, ds, nrows)
self.session.snapshot("drop=(all)")
# Scenario: 2
# Check to see LAS working with old reader
bigvalue2 = "ccccc" * 100
session2 = self.conn.open_session()
session2.begin_transaction('isolation=snapshot')
self.large_updates(self.session, uri, bigvalue2, ds, nrows)
# Check to see the value after recovery
self.durable_check(bigvalue2, uri, ds, nrows)
session2.rollback_transaction()
session2.close()
# Scenario: 3
# Check to see LAS working with old timestamp
bigvalue3 = "ddddd" * 100
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1))
self.large_updates(self.session, uri, bigvalue3, ds, nrows, timestamp=True)
# Check to see data can be see only till the stable_timestamp
self.durable_check(bigvalue2, uri, ds, nrows)
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(i + 1))
# Check to see latest data can be seen
self.durable_check(bigvalue3, uri, ds, nrows)
def test_checkpoint_cursor_update(self):
ds = SimpleDataSet(self, self.uri, 100, key_format=self.fmt)
ds.populate()
self.session.checkpoint("name=ckpt")
cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt")
cursor.set_key(ds.key(10))
cursor.set_value("XXX")
msg = "/Unsupported cursor/"
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cursor.insert(), msg)
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cursor.remove(), msg)
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cursor.update(), msg)
cursor.close()
def test_smoke(self):
ds = SimpleDataSet(self, self.uri, self.nentries,
config=self.config, key_format=self.keyfmt)
ds.populate()
self.reopen_conn()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(100))
self.assertEqual(c.search(), 0)
self.assertEqual(c.get_value(), ds.value(100))
c.set_key(ds.key(101))
self.assertEqual(c.search(), 0)
self.assertEqual(c.get_value(), ds.value(101))
c.set_key(ds.key(9999))
self.assertEqual(c.search(), 0)
self.assertEqual(c.get_value(), ds.value(9999))
def test_modify_delete(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format='u')
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
self.assertEquals(c.remove(), 0)
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(10))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
def test_duplicate_cursor(self):
uri = self.uri + self.name
# A simple, one-file file or table object.
ds = SimpleDataSet(self, uri, self.nentries, key_format=self.fmt)
ds.populate()
self.iterate(uri, ds)
self.session.drop(uri, None)
# A complex, multi-file table object.
if self.uri == "table:":
ds = ComplexDataSet(self, uri, self.nentries, key_format=self.fmt)
ds.populate()
self.iterate(uri, ds)
self.session.drop(uri, None)
def test_reconfig_fail(self):
uri = 'table:reconfig_fail'
ds = SimpleDataSet(self, uri, 100, key_format='S')
ds.populate()
self.session.begin_transaction("isolation=snapshot")
c = self.session.open_cursor(uri, None)
c.set_key(ds.key(20))
c.set_value("abcde")
self.assertEquals(c.update(), 0)
compat_str = 'compatibility=(release="3.0.0")'
msg = '/system must be quiescent/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda:self.conn.reconfigure(compat_str), msg)
def test_insert_over_delete_replace(self):
msg = '/WT_CACHE_FULL.*/'
ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt,
value_format=self.valuefmt, config=self.table_config)
self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
lambda:ds.populate(), msg)
cursor = self.session.open_cursor(self.uri, None)
cursor.prev()
last_key = int(cursor.get_key())
# Now that the database contains as much data as will fit into
# the configured cache, verify removes succeed.
cursor = self.session.open_cursor(self.uri, None)
for i in range(1, last_key / 4, 1):
cursor.set_key(ds.key(i))
cursor.remove()
cursor.reset()
# Spin inserting to give eviction a chance to reclaim space
inserted = False
for i in range(1, 1000):
try:
cursor[ds.key(1)] = ds.value(1)
except wiredtiger.WiredTigerError:
cursor.reset()
sleep(1)
continue
inserted = True
break
self.assertTrue(inserted)
def test_hazard(self):
uri = "table:hazard"
ds = SimpleDataSet(self, uri, 1000)
ds.populate()
# Open 10,000 cursors and pin a page to set a hazard pointer.
cursors = []
for i in range(0, 10000):
c = self.session.open_cursor(uri, None)
c.set_key(ds.key(10))
c.search()
cursors.append(c)
# Close the cursors, clearing the hazard pointer.
for c in cursors:
c.close()
def test_modify_smoke_reopen(self):
if self.skip():
return
ds = SimpleDataSet(self,
self.uri,
100,
key_format=self.keyfmt,
value_format='u')
ds.populate()
self.modify_load(ds, False)
# Flush to disk, forcing reconciliation.
self.reopen_conn()
self.modify_confirm(ds, False)
def test_truncate_cursor_notset(self):
uri = self.type + self.name
msg = '/requires key be set/'
ds = SimpleDataSet(self, uri, 100)
ds.populate()
c1 = self.session.open_cursor(uri, None)
c2 = self.session.open_cursor(uri, None)
c2.set_key(ds.key(10))
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.session.truncate(None, c1, c2, None), msg)
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.session.truncate(None, c2, c1, None), msg)
c1.close()
c2.close()
def test_bug024(self):
nrows = 10
ds = SimpleDataSet(self,
self.uri,
nrows,
key_format="S",
value_format='u')
ds.populate()
self.conn.close()
# Copying the file manually to recreate the issue described in WT-6526.
shutil.copy('WiredTiger.turtle', 'WiredTiger.turtle.set')
# Open wiredtiger in new directory and in readonly mode.
conn = self.wiredtiger_open(self.home, "readonly")
conn.close()
def test_truncate_cursor_order(self):
uri = self.type + self.name
ds = SimpleDataSet(self, uri, 100, key_format=self.keyfmt)
ds.populate()
c1 = self.session.open_cursor(uri, None)
c2 = self.session.open_cursor(uri, None)
c1.set_key(ds.key(20))
c2.set_key(ds.key(10))
msg = '/the start cursor position is after the stop cursor position/'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.truncate(None, c1, c2, None), msg)
c1.set_key(ds.key(10))
c2.set_key(ds.key(20))
self.session.truncate(None, c1, c2, None)
def test_hazard(self):
uri = "table:hazard"
ds = SimpleDataSet(self, uri, 1000)
ds.populate()
# Open 10,000 cursors and pin a page to set a hazard pointer.
cursors = []
for i in range(0, 10000):
c = self.session.open_cursor(uri, None)
c.set_key(ds.key(10))
c.search()
cursors.append(c)
# Close the cursors, clearing the hazard pointer.
for c in cursors:
c.close()
def test_reconfig_fail(self):
uri = 'table:reconfig_fail'
ds = SimpleDataSet(self, uri, 100, key_format='S')
ds.populate()
self.session.begin_transaction("isolation=snapshot")
c = self.session.open_cursor(uri, None)
c.set_key(ds.key(20))
c.set_value("abcde")
self.assertEquals(c.update(), 0)
compat_str = 'compatibility=(release="3.0.0")'
msg = '/system must be quiescent/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.conn.reconfigure(compat_str),
msg)
def test_sharing(self):
# FIXME: WT-8235 Enable the test once file containing transaction ids is supported.
self.skipTest('Sharing the checkpoint file containing transaction ids is not supported')
ds = SimpleDataSet(self, self.uri, 10)
ds.populate()
ds.check()
self.session.checkpoint()
ds.check()
# Create a secondary database
dir2 = os.path.join(self.home, 'SECONDARY')
os.mkdir(dir2)
conn2 = self.setUpConnectionOpen(dir2)
session2 = conn2.open_session()
# Reference the tree from the secondary:
metac = self.session.open_cursor('metadata:')
metac2 = session2.open_cursor('metadata:', None, 'readonly=0')
uri2 = self.uri[:5] + '../' + self.uri[5:]
metac2[uri2] = metac[self.uri] + ",readonly=1"
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
newds = SimpleDataSet(self, self.uri, 10000)
newds.populate()
newds.check()
self.session.checkpoint()
newds.check()
# Check we can still read from the last checkpoint
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
# Bump to new checkpoint
origmeta = metac[self.uri]
checkpoint = re.search(r',checkpoint=\(.+?\)\)', origmeta).group(0)[1:]
self.pr('Orig checkpoint: ' + checkpoint)
session2.alter(uri2, checkpoint)
self.pr('New metadata on secondaery: ' + metac2[uri2])
# Check that we can see the new data
cursor2 = session2.open_cursor(uri2)
newds.check_cursor(cursor2)
def test_rollback_to_stable_with_stable_remove(self):
nrows = 1000
# Prepare transactions for column store table is not yet supported.
if self.prepare and self.key_format == 'r':
self.skipTest('Prepare transactions for column store table is not yet supported')
# Create a table without logging.
uri = "table:rollback_to_stable13"
ds = SimpleDataSet(
self, uri, 0, key_format=self.key_format, value_format="S", config='split_pct=50,log=(enabled=false)')
ds.populate()
# Pin oldest and stable to timestamp 10.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10) +
',stable_timestamp=' + timestamp_str(10))
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
value_c = "ccccc" * 100
# Perform several updates.
self.large_updates(uri, value_a, ds, nrows, self.prepare, 20)
# Perform several updates.
self.large_updates(uri, value_b, ds, nrows, self.prepare, 30)
# Perform several removes.
self.large_removes(uri, ds, nrows, self.prepare, 40)
# Pin stable to timestamp 50 if prepare otherwise 40.
if self.prepare:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(50))
else:
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(40))
# Perform several updates and checkpoint.
self.large_updates(uri, value_c, ds, nrows, self.prepare, 60)
self.session.checkpoint()
# Verify data is visible and correct.
self.check(value_a, uri, nrows, 20)
self.check(None, uri, 0, 40)
self.check(value_c, uri, nrows, 60)
self.conn.rollback_to_stable()
# Perform several updates and checkpoint.
self.large_updates(uri, value_c, ds, nrows, self.prepare, 60)
self.session.checkpoint()
# Simulate a server crash and restart.
simulate_crash_restart(self, ".", "RESTART")
# Check that the correct data is seen at and after the stable timestamp.
self.check(None, uri, 0, 50)
# Check that we restore the correct value from the history store.
self.check(value_a, uri, nrows, 20)
stat_cursor = self.session.open_cursor('statistics:', None, None)
restored_tombstones = stat_cursor[stat.conn.txn_rts_hs_restore_tombstones][2]
self.assertEqual(restored_tombstones, nrows)
def test_sharing(self):
args = 'block_allocation=log-structured'
self.verbose(3,
'Test log-structured allocation with config: ' + args + ' count: ' + str(self.nrecs))
ds = SimpleDataSet(self, self.uri, 10, config=args)
ds.populate()
ds.check()
self.session.checkpoint()
ds.check()
# Create a secondary database
dir2 = os.path.join(self.home, 'SECONDARY')
os.mkdir(dir2)
conn2 = self.setUpConnectionOpen(dir2)
session2 = conn2.open_session()
# Reference the tree from the secondary:
metac = self.session.open_cursor('metadata:')
metac2 = session2.open_cursor('metadata:', None, 'readonly=0')
uri2 = self.uri[:5] + '../' + self.uri[5:]
metac2[uri2] = metac[self.uri] + ",readonly=1"
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
newds = SimpleDataSet(self, self.uri, 10000, config=args)
newds.populate()
newds.check()
self.session.checkpoint()
newds.check()
# Check we can still read from the last checkpoint
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
# Bump to new checkpoint
origmeta = metac[self.uri]
checkpoint = re.search(r',checkpoint=\(.+?\)\)', origmeta).group(0)[1:]
self.pr('Orig checkpoint: ' + checkpoint)
session2.alter(uri2, checkpoint)
self.pr('New metadata on secondaery: ' + metac2[uri2])
# Check that we can see the new data
cursor2 = session2.open_cursor(uri2)
newds.check_cursor(cursor2)
def test_bug(self):
uri = "table:bug026"
nrows = 1000
ntxns = 500
ds = SimpleDataSet(self,
uri,
0,
key_format=self.key_format,
value_format=self.value_format,
config='log=(enabled=false)')
ds.populate()
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
value_c = "ccccc" * 100
# Write some data.
cursor = self.session.open_cursor(uri)
self.session.begin_transaction()
for i in range(1, nrows + 1):
cursor[ds.key(i)] = value_a
self.session.commit_transaction()
self.session.checkpoint()
# Create a bunch of transactions and leave all but one hanging.
sessions = {}
cursors = {}
for i in range(1, ntxns + 1):
sessions[i] = self.conn.open_session()
cursors[i] = sessions[i].open_cursor(uri)
sessions[i].begin_transaction()
cursors[i][ds.key(i)] = value_b
self.session.begin_transaction()
cursor[ds.key(nrows)] = value_c
self.session.commit_transaction()
self.session.checkpoint()
# Should not see value_b.
self.check(ds, nrows, value_a, value_c)
# Now crash.
simulate_crash_restart(self, ".", "RESTART")
# Should still not see value_b.
self.check(ds, nrows, value_a, value_c)
def test_checkpoint(self):
uri = 'table:checkpoint24'
nrows = 10000
# Create a table.
ds = SimpleDataSet(self,
uri,
0,
key_format=self.key_format,
value_format=self.value_format,
config=self.extraconfig)
ds.populate()
if self.value_format == '8t':
value_a = 97
else:
value_a = "aaaaa" * 100
# Write some data at time 10.
self.large_updates(uri, ds, nrows, value_a)
# Reopen the connection (which checkpoints it) so it's all on disk and not in memory.
self.reopen_conn()
# Truncate half of it.
self.do_truncate(ds, nrows // 4 + 1, nrows // 4 + nrows // 2)
# Check stats to make sure we fast-deleted at least one page.
# (Except for FLCS, where it's not supported and we should fast-delete zero pages.)
stat_cursor = self.session.open_cursor('statistics:', None, None)
fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
if self.value_format == '8t':
self.assertEqual(fastdelete_pages, 0)
else:
self.assertGreater(fastdelete_pages, 0)
# Take a checkpoint.
self.do_checkpoint(self.first_checkpoint)
if self.do_reopen:
self.reopen_conn()
# Read the checkpoint.
nonzeros = nrows // 2
zeros = nrows - nonzeros
self.check(ds, self.first_checkpoint, nonzeros, zeros, value_a)
def test_duplicate_cursor(self):
uri = self.uri + self.name
# A simple, one-file file or table object.
ds = SimpleDataSet(self, uri, self.nentries, \
key_format=self.keyfmt, value_format=self.valfmt)
ds.populate()
self.iterate(uri, ds)
self.dropUntilSuccess(self.session, uri)
# A complex, multi-file table object.
if self.uri == "table:" and self.valfmt != '8t':
ds = ComplexDataSet(self, uri, self.nentries, \
key_format=self.keyfmt, value_format=self.valfmt)
ds.populate()
self.iterate(uri, ds)
self.dropUntilSuccess(self.session, uri)
def test_checkpoint(self):
uri = 'table:checkpoint12'
nrows = 1000
# Create a table.
ds = SimpleDataSet(
self, uri, 0, key_format=self.key_format, value_format=self.value_format)
ds.populate()
if self.value_format == '8t':
value_a = 97
value_b = 98
value_c = 99
else:
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
value_c = "ccccc" * 100
# Pin oldest and stable timestamps to 5.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5) +
',stable_timestamp=' + self.timestamp_str(5))
# Write some data at time 10.
self.large_updates(uri, ds, nrows, value_a, 10)
# Make a checkpoint.
self.session.checkpoint()
# Write some more data at time 20.
self.large_updates(uri, ds, nrows, value_a, 20)
# Open the checkpoint.
ckpt_cursor = self.session.open_cursor(uri, None, 'checkpoint=WiredTigerCheckpoint')
ckpt_cursor.set_key(ds.key(1))
# Write some further data, and prepare it at time 30.
cursor = self.session.open_cursor(uri)
self.session.begin_transaction()
for i in range(1, nrows // 2):
cursor[ds.key(i)] = value_b
self.session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(30))
# Now try reading the checkpoint.
msg = '/Invalid argument/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.operate(ckpt_cursor), msg)
def test_checkpoint_snapshot(self):
self.moresetup()
ds = SimpleDataSet(self, self.uri, 0, \
key_format=self.key_format, value_format=self.value_format, \
config='log=(enabled=false)'+self.extraconfig)
ds.populate()
self.large_updates(self.uri, self.valuea, ds, self.nrows, 0)
self.check(self.valuea, self.uri, self.nrows, 0, False)
session1 = self.conn.open_session()
session1.begin_transaction()
cursor1 = session1.open_cursor(self.uri)
for i in range(self.nrows+1, (self.nrows*2)+1):
cursor1.set_key(ds.key(i))
cursor1.set_value(self.valuea)
self.assertEqual(cursor1.insert(), 0)
# Create a checkpoint thread
done = threading.Event()
ckpt = checkpoint_thread(self.conn, done)
try:
ckpt.start()
# Sleep for sometime so that checkpoint starts before committing last transaction.
time.sleep(2)
session1.commit_transaction()
finally:
done.set()
ckpt.join()
#Simulate a crash by copying to a new directory(RESTART).
simulate_crash_restart(self, ".", "RESTART")
# Check the table contains the last checkpointed value.
self.check(self.valuea, self.uri, self.nrows, 0, True)
stat_cursor = self.session.open_cursor('statistics:', None, None)
inconsistent_ckpt = stat_cursor[stat.conn.txn_rts_inconsistent_ckpt][2]
keys_removed = stat_cursor[stat.conn.txn_rts_keys_removed][2]
stat_cursor.close()
self.assertGreater(inconsistent_ckpt, 0)
self.assertGreaterEqual(keys_removed, 0)
def test_wedge(self):
# Try to really wedge the cache full
ds = SimpleDataSet(self,
self.uri,
0,
key_format=self.keyfmt,
value_format=self.valuefmt,
config=self.table_config)
ds.populate()
cursor = self.session.open_cursor(self.uri, None)
run = 0
start, last_key = -1000, 0
while last_key - start > 100:
msg = '/WT_CACHE_FULL.*/'
start = last_key
self.assertRaisesHavingMessage(
wiredtiger.WiredTigerError,
lambda: self.fill(cursor, ds, start, 10000000), msg)
cursor.reset()
sleep(1)
# Figure out the last key we successfully inserted, and check all
# previous inserts are still there.
cursor.prev()
last_key = int(cursor.get_key())
run += 1
self.pr('Finished iteration ' + str(run) + ', last_key = ' +
str(last_key))
self.pr('Checking ' + str(last_key) + ' keys')
ds = SimpleDataSet(self,
self.uri,
last_key,
key_format=self.keyfmt,
value_format=self.valuefmt,
config=self.table_config)
# This test is *much* slower for fixed-length column stores: we fit
# many more records into the cache, so don't do as many passes through
# the data.
checks = 10 if self.valuefmt.endswith('t') else 100
for run in range(checks):
ds.check()
self.pr('Finished check ' + str(run))
sleep(1)
def test_basic_conn_stats(self):
# Build an object and force some writes.
ds = SimpleDataSet(self, self.uri, 1000,
config=self.config, key_format=self.keyfmt, value_format = self.valfmt)
ds.populate()
self.session.checkpoint(None)
# See that we can get a specific stat value by its key and verify its
# entry is self-consistent.
allstat_cursor = self.session.open_cursor('statistics:', None, None)
self.check_stats(allstat_cursor, 10, 'block-manager: blocks written')
values = allstat_cursor[stat.conn.block_write]
self.assertEqual(values[0], 'block-manager: blocks written')
val = self.statstr_to_int(values[1])
self.assertEqual(val, values[2])
allstat_cursor.close()
def test_checkpoint(self):
uri = 'table:checkpoint17'
nrows = 1000
# Create a table.
ds = SimpleDataSet(self,
uri,
0,
key_format=self.key_format,
value_format=self.value_format,
config=self.extraconfig)
ds.populate()
if self.value_format == '8t':
value_a = 97
value_b = 98
value_c = 99
value_d = 100
else:
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
value_c = "ccccc" * 100
value_d = "ddddd" * 100
# Set oldest and stable to 5.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5) +
',stable_timestamp=' + self.timestamp_str(5))
# Write some history and checkpoint it.
self.large_updates(ds, 1, nrows + 1, value_a, 10)
self.large_updates(ds, 1, nrows + 1, value_b, 20)
self.large_updates(ds, 1, nrows + 1, value_c, 30)
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(30))
self.session.checkpoint()
# Write some disjoint data that should not generate more history.
self.large_updates(ds, nrows + 1, 2 * nrows + 1, value_d, 40)
# Mark this data stable and checkpoint it.
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(40))
self.do_checkpoint(self.second_checkpoint)
# Make sure we can still read the history.
self.check(ds, self.second_checkpoint, nrows, value_a, nrows, 10)
self.check(ds, self.second_checkpoint, nrows, value_b, nrows, 20)
self.check(ds, self.second_checkpoint, nrows, value_c, nrows, 30)
def test_checkpoint(self):
uri1 = 'table:checkpoint16a'
uri2 = 'table:checkpoint16b'
nrows = 1000
# Create two tables.
ds1 = SimpleDataSet(self,
uri1,
0,
key_format=self.key_format,
value_format=self.value_format,
config=self.extraconfig)
ds1.populate()
ds2 = SimpleDataSet(self,
uri2,
0,
key_format=self.key_format,
value_format=self.value_format,
config=self.extraconfig)
ds2.populate()
if self.value_format == '8t':
value_a = 97
value_b = 98
else:
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
# Set oldest and stable to 5.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5) +
',stable_timestamp=' + self.timestamp_str(5))
# Write some data to both tables and checkpoint it.
self.large_updates(ds1, nrows, value_a)
self.large_updates(ds2, nrows, value_a)
self.session.checkpoint()
# Write some more data but only to table 2.
self.large_updates(ds2, nrows, value_b)
# Checkpoint this data.
self.do_checkpoint(self.second_checkpoint)
# Make sure we can read table 1 from the second checkpoint.
self.check(ds1, self.second_checkpoint, nrows, value_a)
def test_modify_abort(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
# Start a transaction.
self.session.begin_transaction()
# Insert a new record.
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(30))
c.set_value(ds.value(30))
self.assertEquals(c.insert(), 0)
# Test that we can successfully modify our own record.
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), 0)
# Test that another transaction cannot modify our uncommitted record.
xs = self.conn.open_session()
xc = xs.open_cursor(self.uri, None)
xs.begin_transaction()
xc.set_key(ds.key(30))
xc.set_value(ds.value(30))
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
xc.set_key(ds.key(30))
self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND)
xs.rollback_transaction()
# Rollback our transaction.
self.session.rollback_transaction()
# Test that we can't modify our aborted insert.
self.session.begin_transaction()
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
self.session.rollback_transaction()
def test_checkpoint_hs_reads(self):
# Create a small table.
uri = "table:test_hs03"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format=self.key_format, value_format=self.value_format)
ds.populate()
if self.value_format == '8t':
bigvalue = 97
bigvalue2 = 100
else:
bigvalue = b"aaaaa" * 100
bigvalue2 = b"ddddd" * 100
# Initially load huge data.
cursor = self.session.open_cursor(uri)
for i in range(1, 10000):
cursor[ds.key(nrows + i)] = bigvalue
cursor.close()
self.session.checkpoint()
# Check to see the history store working with old timestamp.
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1))
hs_writes_start = self.get_stat(stat.conn.cache_write_hs)
self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000)
# If the test sizing is correct, the history will overflow the cache.
self.session.checkpoint()
hs_writes = self.get_stat(stat.conn.cache_write_hs) - hs_writes_start
self.assertGreaterEqual(hs_writes, 0)
for ts in range(2, 4):
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(ts))
# Now just update one record and checkpoint again.
self.large_updates(self.session, uri, bigvalue2, ds, nrows, 1)
hs_reads_start = self.get_stat(stat.conn.cache_hs_read)
self.session.checkpoint()
hs_reads = self.get_stat(stat.conn.cache_hs_read) - hs_reads_start
# Since we're dealing with eviction concurrent with checkpoints
# and skewing is controlled by a heuristic, we can't put too tight
# a bound on this.
self.assertLessEqual(hs_reads, 200)
def test_txn(self):
nrows = 2000
# Create a table.
uri_1 = "table:txn23_1"
ds_1 = SimpleDataSet(
self, uri_1, 0, key_format=self.key_format, value_format="S")
ds_1.populate()
# Create another table.
uri_2 = "table:txn23_2"
ds_2 = SimpleDataSet(
self, uri_2, 0, key_format=self.key_format, value_format="S")
ds_2.populate()
# Pin oldest and stable to timestamp 10.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(10) +
',stable_timestamp=' + self.timestamp_str(10))
value_a = "aaaaa" * 100
value_b = "bbbbb" * 100
value_c = "ccccc" * 100
value_d = "ddddd" * 100
# Perform several updates.
self.large_updates(uri_1, value_d, ds_1, nrows, 20)
self.large_updates(uri_1, value_c, ds_1, nrows, 30)
self.large_updates(uri_1, value_b, ds_1, nrows, 40)
self.large_updates(uri_1, value_a, ds_1, nrows, 50)
self.large_updates(uri_2, value_d, ds_2, nrows, 20)
self.large_updates(uri_2, value_c, ds_2, nrows, 30)
self.large_updates(uri_2, value_b, ds_2, nrows, 40)
self.large_updates(uri_2, value_a, ds_2, nrows, 50)
# Verify data is visible and correct.
self.check(value_d, uri_1, ds_1, nrows, 20)
self.check(value_c, uri_1, ds_1, nrows, 30)
self.check(value_b, uri_1, ds_1, nrows, 40)
self.check(value_a, uri_1, ds_1, nrows, 50)
self.check(value_d, uri_2, ds_2, nrows, 20)
self.check(value_c, uri_2, ds_2, nrows, 30)
self.check(value_b, uri_2, ds_2, nrows, 40)
self.check(value_a, uri_2, ds_2, nrows, 50)
def test_rollback_to_stable(self):
nrows = 1000
nds = 10
# Create a few tables and populate them with some initial data.
#
# Our way of preventing history store operations from interfering with rollback to stable's
# transaction check is by draining active evictions from each open dhandle.
#
# It's important that we have a few different tables to work with so that it's
# representative of a real situation. But also don't make the number too high relative to
# the number of updates or we may not have history for any of the tables.
ds_list = list()
for i in range(0, nds):
uri = 'table:rollback_to_stable22_{}'.format(i)
ds = SimpleDataSet(self,
uri,
0,
key_format='i',
value_format='S',
config='log=(enabled=false)')
ds.populate()
ds_list.append(ds)
self.assertEqual(len(ds_list), nds)
# 100 bytes of data are being inserted into 1000 rows.
# This happens 1000 iterations.
# Overall, that's 100MB of data which is guaranteed to kick start eviction.
for i in range(1, 1000):
# Generate a value, timestamp and table based off the index.
value = str(i)[0] * 100
ts = i * 10
ds = ds_list[i % nds]
# Perform updates.
self.large_updates(ds.uri, value, ds, nrows, False, ts)
# Every hundred updates, let's run rollback to stable. This is likely to happen during
# a history store eviction at least once.
if i % 100 == 0:
# Put the timestamp backwards so we can rollback the updates we just did.
stable_ts = (i - 1) * 10
self.conn.set_timestamp('stable_timestamp=' +
self.timestamp_str(stable_ts))
self.conn.rollback_to_stable()
def address_deleted(self):
# Create the object, force it to disk, and verify the object.
ds = SimpleDataSet(self, self.uri, self.nentries, config=self.config)
ds.populate()
self.reopen_conn()
self.session.verify(self.uri)
# Create a new session and start a transaction to force the upcoming
# checkpoint operation to write address-deleted cells to disk.
tmp_session = self.conn.open_session(None)
tmp_session.begin_transaction("isolation=snapshot")
# Truncate a big range of rows; the leaf pages aren't in memory, so
# leaf page references will be deleted without being read.
start = self.session.open_cursor(self.uri, None)
start.set_key(ds.key(10))
end = self.session.open_cursor(self.uri, None)
end.set_key(ds.key(self.nentries - 10))
self.session.truncate(None, start, end, None)
self.assertEqual(start.close(), 0)
self.assertEqual(end.close(), 0)
# Checkpoint, forcing address-deleted cells to be written.
self.session.checkpoint()
# Crash/reopen the connection and verify the object.
self.reopen_conn()
self.session.verify(self.uri)
# Open a cursor and update a record (to dirty the tree, else we won't
# mark pages with address-deleted cells dirty), then walk the tree so
# we get a good look at all the internal pages and the address-deleted
# cells.
cursor = self.session.open_cursor(self.uri, None)
cursor.set_key(ds.key(5))
cursor.set_value("changed value")
self.assertEqual(cursor.update(), 0)
cursor.reset()
for key,val in cursor:
continue
self.assertEqual(cursor.close(), 0)
# Checkpoint, freeing the pages.
self.session.checkpoint()
return ds
def test_checkpoint_las_reads(self):
if not wiredtiger.timestamp_build():
self.skipTest('requires a timestamp build')
# Create a small table.
uri = "table:test_las03"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u')
ds.populate()
bigvalue = "aaaaa" * 100
# Initially load huge data
cursor = self.session.open_cursor(uri)
for i in range(1, 10000):
cursor[ds.key(nrows + i)] = bigvalue
cursor.close()
self.session.checkpoint()
# Check to see LAS working with old timestamp
bigvalue2 = "ddddd" * 100
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1))
las_writes_start = self.get_stat(stat.conn.cache_write_lookaside)
self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000)
# If the test sizing is correct, the history will overflow the cache
self.session.checkpoint()
las_writes = self.get_stat(
stat.conn.cache_write_lookaside) - las_writes_start
self.assertGreaterEqual(las_writes, 0)
for ts in range(2, 4):
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(ts))
# Now just update one record and checkpoint again
self.large_updates(self.session, uri, bigvalue2, ds, nrows, 1)
las_reads_start = self.get_stat(stat.conn.cache_read_lookaside)
self.session.checkpoint()
las_reads = self.get_stat(
stat.conn.cache_read_lookaside) - las_reads_start
# Since we're dealing with eviction concurrent with checkpoints
# and skewing is controlled by a heuristic, we can't put too tight
# a bound on this.
self.assertLessEqual(las_reads, 100)
def test_las(self):
# Create a small table.
uri = "table:test_las"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format="S")
ds.populate()
# Take a snapshot.
self.session.snapshot("name=xxx")
# Insert a large number of records, we'll hang if the lookaside table
# isn't doing its thing.
c = self.session.open_cursor(uri)
bigvalue = "abcde" * 100
for i in range(1, 1000000):
c.set_key(ds.key(nrows + i))
c.set_value(bigvalue)
self.assertEquals(c.insert(), 0)
def test_modify_delete(self):
ds = SimpleDataSet(self,
self.uri,
20,
key_format=self.keyfmt,
value_format='u')
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
self.assertEquals(c.remove(), 0)
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(10))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
def test_smoke(self):
ds = SimpleDataSet(self,
self.uri,
self.nentries,
config=self.config,
key_format=self.keyfmt)
ds.populate()
self.reopen_conn()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(100))
self.assertEqual(c.search(), 0)
self.assertEqual(c.get_value(), ds.value(100))
c.set_key(ds.key(101))
self.assertEqual(c.search(), 0)
self.assertEqual(c.get_value(), ds.value(101))
c.set_key(ds.key(9999))
self.assertEqual(c.search(), 0)
self.assertEqual(c.get_value(), ds.value(9999))
def test_modify_delete(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
self.assertEquals(c.remove(), 0)
self.session.begin_transaction("isolation=snapshot")
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
mods = self.fix_mods(mods)
c.set_key(ds.key(10))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
self.session.commit_transaction()
def test_las(self):
# Create a small table.
uri = "table:test_las"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format="S")
ds.populate()
# Take a snapshot.
self.session.snapshot("name=xxx")
# Insert a large number of records, we'll hang if the lookaside table
# isn't doing its thing.
c = self.session.open_cursor(uri)
bigvalue = "abcde" * 100
for i in range(1, 1000000):
c.set_key(ds.key(nrows + i))
c.set_value(bigvalue)
self.assertEquals(c.insert(), 0)
def test_modify_abort(self):
ds = SimpleDataSet(self,
self.uri,
20,
key_format=self.keyfmt,
value_format='u')
ds.populate()
# Start a transaction.
self.session.begin_transaction()
# Insert a new record.
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(30))
c.set_value(ds.value(30))
self.assertEquals(c.insert(), 0)
# Test that we can successfully modify our own record.
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), 0)
# Test that another transaction cannot modify our uncommitted record.
xs = self.conn.open_session()
xc = xs.open_cursor(self.uri, None)
xc.set_key(ds.key(30))
xc.set_value(ds.value(30))
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
xc.set_key(ds.key(30))
self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND)
# Rollback our transaction.
self.session.rollback_transaction()
# Test that we can't modify our aborted insert.
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
def test_prepare(self):
# Create a small table.
uri = "table:test"
nrows = 1000
ds = SimpleDataSet(self, uri, 0, key_format="S", value_format='u')
ds.populate()
value_a = b"aaaaa" * 100
value_b = b"bbbbb" * 100
# Commit some updates along with a prepared update, which is not resolved.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(10))
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10))
# Initially load huge data
self.updates(ds, uri, nrows, value_a, 20)
# Add some more updates
self.updates(ds, uri, nrows, value_b, 30)
# Checkpoint
self.session.checkpoint()
# Remove the updates from a prepare session and and keep it open.
session_p = self.conn.open_session()
cursor_p = session_p.open_cursor(uri)
session_p.begin_transaction('isolation=snapshot')
for i in range(1, nrows):
cursor_p.set_key(ds.key(i))
self.assertEquals(cursor_p.remove(), 0)
session_p.prepare_transaction('prepare_timestamp=' + timestamp_str(40))
self.check(ds, uri, nrows, value_a, 20)
self.check(ds, uri, nrows, value_b, 50)
#rollback the prepared session
session_p.rollback_transaction()
self.check(ds, uri, nrows, value_a, 20)
self.check(ds, uri, nrows, value_b, 50)
# close sessions.
cursor_p.close()
session_p.close()
self.session.close()
def test_lsm_drop_active(self):
uri = 'lsm:' + self.name
ds = SimpleDataSet(self, uri, 10000, config=self.config)
ds.populate()
# Force to disk
self.reopen_conn()
# An open cursors should cause failure.
cursor = self.session.open_cursor(uri, None, None)
self.assertRaises(wiredtiger.WiredTigerError,
lambda: self.session.drop(uri, None))
cursor.close()
# Add enough records that a merge should be running
ds = SimpleDataSet(self, uri, 50000, config=self.config)
ds.populate()
# The drop should succeed even when LSM work units are active
self.session.drop(uri)
def test_checkpoint_las_reads(self):
if not wiredtiger.timestamp_build():
self.skipTest('requires a timestamp build')
# Create a small table.
uri = "table:test_las03"
nrows = 100
ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u')
ds.populate()
bigvalue = "aaaaa" * 100
# Initially load huge data
cursor = self.session.open_cursor(uri)
for i in range(1, 10000):
cursor[ds.key(nrows + i)] = bigvalue
cursor.close()
self.session.checkpoint()
# Check to see LAS working with old timestamp
bigvalue2 = "ddddd" * 100
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1))
las_writes_start = self.get_stat(stat.conn.cache_write_lookaside)
self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000)
# If the test sizing is correct, the history will overflow the cache
self.session.checkpoint()
las_writes = self.get_stat(stat.conn.cache_write_lookaside) - las_writes_start
self.assertGreaterEqual(las_writes, 0)
for ts in range(2, 4):
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(ts))
# Now just update one record and checkpoint again
self.large_updates(self.session, uri, bigvalue2, ds, nrows, 1)
las_reads_start = self.get_stat(stat.conn.cache_read_lookaside)
self.session.checkpoint()
las_reads = self.get_stat(stat.conn.cache_read_lookaside) - las_reads_start
# Since we're dealing with eviction concurrent with checkpoints
# and skewing is controlled by a heuristic, we can't put too tight
# a bound on this.
self.assertLessEqual(las_reads, 100)
def test_cursor_random_reasonable_distribution(self):
uri = self.type
num_entries = self.records
if uri == 'table:random':
config = 'leaf_page_max=100MB'
else:
config = ''
# Set the leaf-page-max value, otherwise the page might split.
ds = SimpleDataSet(self, uri, num_entries, config=config)
ds.populate()
# Setup an array to track which keys are seen
visitedKeys = [0] * (num_entries + 1)
# Setup a counter to see when we find a sequential key
sequentialKeys = 0
cursor = self.session.open_cursor(uri, None, 'next_random=true')
lastKey = None
for i in range(0, num_entries):
self.assertEqual(cursor.next(), 0)
current = cursor.get_key()
current = int(current)
visitedKeys[current] = visitedKeys[current] + 1
if lastKey != None:
if current == (lastKey + 1):
sequentialKeys += 1
lastKey = current
differentKeys = sum(x > 0 for x in visitedKeys)
#print visitedKeys
#print differentKeys
'''
self.tty('differentKeys: ' + str(differentKeys) + ' of ' + \
str(num_entries) + ', ' + \
str((int)((differentKeys * 100) / num_entries)) + '%')
'''
# Can't test for non-sequential data when there is 1 item in the table
if num_entries > 1:
self.assertGreater(num_entries - 1, sequentialKeys,
'cursor is returning sequential data')
self.assertGreater(differentKeys, num_entries / 4,
'next_random random distribution not adequate')
def test_modify_many(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format='u')
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
orig = 'abcdefghijklmnopqrstuvwxyz'
c.set_value(orig)
self.assertEquals(c.update(), 0)
for i in range(0, 50000):
new = "".join([random.choice(string.digits) for i in xrange(5)])
orig = orig[:10] + new + orig[15:]
mods = []
mod = wiredtiger.Modify(new, 10, 5)
mods.append(mod)
self.assertEquals(c.modify(mods), 0)
c.set_key(ds.key(10))
self.assertEquals(c.search(), 0)
self.assertEquals(c.get_value(), orig)
def test_checkpoint_target(self):
# Create 3 objects, change one record to an easily recognizable string.
uri = self.uri + '1'
ds1 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
ds1.populate()
self.update(uri, ds1, 'ORIGINAL')
uri = self.uri + '2'
ds2 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
ds2.populate()
self.update(uri, ds2, 'ORIGINAL')
uri = self.uri + '3'
ds3 = SimpleDataSet(self, uri, 100, key_format=self.fmt)
ds3.populate()
self.update(uri, ds3, 'ORIGINAL')
# Checkpoint all three objects.
self.session.checkpoint("name=checkpoint-1")
# Update all 3 objects, then checkpoint two of the objects with the
# same checkpoint name.
self.update(self.uri + '1', ds1, 'UPDATE')
self.update(self.uri + '2', ds2, 'UPDATE')
self.update(self.uri + '3', ds3, 'UPDATE')
target = 'target=("' + self.uri + '1"' + ',"' + self.uri + '2")'
self.session.checkpoint("name=checkpoint-1," + target)
# Confirm the checkpoint has the old value in objects that weren't
# checkpointed, and the new value in objects that were checkpointed.
self.check(self.uri + '1', ds1, 'UPDATE')
self.check(self.uri + '2', ds2, 'UPDATE')
self.check(self.uri + '3', ds3, 'ORIGINAL')
def test_search_duplicate(self):
if self.colvar == 0:
return
# Populate the tree.
ds = SimpleDataSet(self, self.uri, 105, key_format=self.key_format,
value_format=self.value_format)
ds.populate()
# Set up deleted records before and after a set of duplicate records,
# and make sure search/search-near returns the correct record.
cursor = self.session.open_cursor(self.uri, None)
for i in range(20, 100):
cursor[ds.key(i)] = '=== IDENTICAL VALUE ==='
for i in range(15, 25):
cursor.set_key(ds.key(i))
self.assertEqual(cursor.remove(), 0)
for i in range(95, 106):
cursor.set_key(ds.key(i))
self.assertEqual(cursor.remove(), 0)
cursor.close()
# Reopen the connection, forcing it to disk and moving the records to
# an on-page format.
self.reopen_conn()
# Open a cursor.
cursor = self.session.open_cursor(self.uri, None)
# Search-near for a record in the deleted set before the duplicate set,
# which should succeed, returning the first record in the duplicate set.
cursor.set_key(ds.key(18))
self.assertEqual(cursor.search_near(), 1)
self.assertEqual(cursor.get_key(), ds.key(25))
# Search-near for a record in the deleted set after the duplicate set,
# which should succeed, returning the last record in the duplicate set.
cursor.set_key(ds.key(98))
self.assertEqual(cursor.search_near(), -1)
self.assertEqual(cursor.get_key(), ds.key(94))
