def test_hs_instantiated_modify(self):
# Create a small table.
uri = "table:test_hs06"
create_params = 'key_format={},value_format=S'.format(self.key_format)
self.session.create(uri, create_params)
value1 = 'a' * 500
value2 = 'b' * 500
# Load 5Mb of data.
self.conn.set_timestamp(
'oldest_timestamp=' + timestamp_str(1) + ',stable_timestamp=' + timestamp_str(1))
cursor = self.session.open_cursor(uri)
for i in range(1, 10000):
self.session.begin_transaction()
cursor[self.create_key(i)] = value1
self.session.commit_transaction('commit_timestamp=' + timestamp_str(2))
# Apply three sets of modifies.
for i in range(1, 11):
self.session.begin_transaction()
cursor.set_key(self.create_key(i))
self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' + timestamp_str(3))
for i in range(1, 11):
self.session.begin_transaction()
cursor.set_key(self.create_key(i))
self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' + timestamp_str(4))
# Since the stable timestamp is still at 1, there will be no birthmark record.
# History store instantiation should choose this update since it is the most recent.
# We want to check that it gets converted into a standard update as appropriate.
for i in range(1, 11):
self.session.begin_transaction()
cursor.set_key(self.create_key(i))
self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' + timestamp_str(5))
# Make a bunch of updates to another table to flush everything out of cache.
uri2 = 'table:test_hs06_extra'
self.session.create(uri2, create_params)
cursor2 = self.session.open_cursor(uri2)
for i in range(1, 10000):
self.session.begin_transaction()
cursor2[self.create_key(i)] = value2
self.session.commit_transaction('commit_timestamp=' + timestamp_str(6))
expected = list(value1)
expected[100] = 'B'
expected[200] = 'C'
expected[300] = 'D'
expected = str().join(expected)
# Go back and read. We should get the initial value with the 3 modifies applied on top.
self.session.begin_transaction('read_timestamp=' + timestamp_str(5))
for i in range(1, 11):
self.assertEqual(cursor[self.create_key(i)], expected)
self.session.rollback_transaction()
def test_reverse_modifies_constructed_after_eviction(self):
uri = "table:test_hs13"
create_params = 'value_format=S,key_format={}'.format(self.key_format)
value1 = 'a' * 10000
value2 = 'b' * 10000
value3 = 'e' * 10000
self.session.create(uri, create_params)
cursor = self.session.open_cursor(uri)
session2 = self.setUpSessionOpen(self.conn)
cursor2 = session2.open_cursor(uri)
# Insert a full value.
self.session.begin_transaction()
cursor[1] = value1
self.session.commit_transaction()
# Insert a modify.
self.session.begin_transaction()
cursor.set_key(1)
cursor.modify([wiredtiger.Modify('A', 0, 0)])
self.session.commit_transaction()
# Validate that we do see the correct value.
session2.begin_transaction()
cursor2.set_key(1)
cursor2.search()
self.assertEquals(cursor2.get_value(), 'A' + value1)
session2.commit_transaction()
# Reset the cursor.
cursor2.reset()
# Begin session2's transaction so it gets a snapshot prior to the full value being
# inserted below.
session2.begin_transaction()
# Insert a second modify.
self.session.begin_transaction()
cursor.set_key(1)
cursor.modify([wiredtiger.Modify('B', 1, 0)])
self.session.commit_transaction()
# Insert one more value.
self.session.begin_transaction()
cursor.set_key(1)
cursor[1] = value2
self.session.commit_transaction()
# Insert a whole bunch of data into the table to force wiredtiger to evict data
# from the previous table.
self.session.begin_transaction()
for i in range(2, 10000):
cursor[i] = value3
self.session.commit_transaction()
# Try to find the value we saw earlier.
cursor2.set_key(1)
cursor2.search()
self.assertEquals(cursor2.get_value(), 'A' + value1)
def test_reverse_modifies_constructed_after_eviction(self):
uri = "table:test_hs13"
create_params = 'value_format=S,key_format={}'.format(self.key_format)
value1 = 'a' * 10000
value2 = 'b' * 10000
value3 = 'e' * 10000
self.session.create(uri, create_params)
cursor = self.session.open_cursor(uri)
session2 = self.setUpSessionOpen(self.conn)
cursor2 = session2.open_cursor(uri)
# Insert a full value.
self.session.begin_transaction()
cursor[1] = value1
self.session.commit_transaction()
# Insert a modify.
self.session.begin_transaction()
cursor.set_key(1)
cursor.modify([wiredtiger.Modify('A', 0, 0)])
self.session.commit_transaction()
# Validate that we do see the correct value.
session2.begin_transaction()
cursor2.set_key(1)
cursor2.search()
self.assertEquals(cursor2.get_value(), 'A' + value1)
session2.commit_transaction()
# Reset the cursor.
cursor2.reset()
# Begin session2's transaction so it gets a snapshot prior to the full value being
# inserted below.
session2.begin_transaction()
# Insert a second modify.
self.session.begin_transaction()
cursor.set_key(1)
cursor.modify([wiredtiger.Modify('B', 1, 0)])
self.session.commit_transaction()
# Insert one more value.
self.session.begin_transaction()
cursor.set_key(1)
cursor[1] = value2
self.session.commit_transaction()
# Configure debug behavior on a cursor to evict the positioned page on cursor reset
# and evict the page.
evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)")
evict_cursor.set_key(1)
self.assertEquals(evict_cursor.search(), 0)
evict_cursor.reset()
# Try to find the value we saw earlier.
cursor2.set_key(1)
cursor2.search()
self.assertEquals(cursor2.get_value(), 'A' + value1)
def test_modify_txn_api(self):
ds = SimpleDataSet(self,
self.uri,
100,
key_format=self.keyfmt,
value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
msg = '/not supported/'
self.session.begin_transaction("isolation=read-uncommitted")
mods = []
mods.append(wiredtiger.Modify('-', 1, 1))
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c.modify(mods), msg)
self.session.rollback_transaction()
self.session.begin_transaction("isolation=read-committed")
mods = []
mods.append(wiredtiger.Modify('-', 1, 1))
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c.modify(mods), msg)
self.session.rollback_transaction()
def test_hs20(self):
uri = 'table:test_hs20'
# Set a very small maximum leaf value to trigger writing overflow values
self.session.create(uri,
'key_format=S,value_format=S,leaf_value_max=10B')
cursor = self.session.open_cursor(uri)
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) +
',stable_timestamp=' + timestamp_str(1))
value1 = 'a' * 500
value2 = 'b' * 50
# Insert a value that is larger than the maximum leaf value.
for i in range(0, 10):
self.session.begin_transaction()
cursor[str(i)] = value1
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(2))
# Do 2 modifies.
for i in range(0, 10):
self.session.begin_transaction()
cursor.set_key(str(i))
mods = [wiredtiger.Modify('B', 500, 1)]
self.assertEqual(cursor.modify(mods), 0)
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(3))
for i in range(0, 10):
self.session.begin_transaction()
cursor.set_key(str(i))
mods = [wiredtiger.Modify('C', 501, 1)]
self.assertEqual(cursor.modify(mods), 0)
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(4))
# Insert more data to trigger eviction.
for i in range(10, 100000):
self.session.begin_transaction()
cursor[str(i)] = value2
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(5))
# Update the overflow values.
for i in range(0, 10):
self.session.begin_transaction()
cursor[str(i)] = value2
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(5))
# Do a checkpoint to move the overflow values to the history store but keep the current in memory disk image.
self.session.checkpoint()
# Search the first modifies.
for i in range(0, 10):
self.session.begin_transaction('read_timestamp=' +
timestamp_str(3))
self.assertEqual(cursor[str(i)], value1 + "B")
self.session.rollback_transaction()
def test_hs20(self):
uri = 'table:test_hs20'
key_format = 'key_format=' + self.key_format
# Set a very small maximum leaf value to trigger writing overflow values
self.session.create(
uri, '{},value_format=S,leaf_value_max=10B'.format(key_format))
cursor = self.session.open_cursor(uri)
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1) +
',stable_timestamp=' + self.timestamp_str(1))
value1 = 'a' * 500
value2 = 'b' * 50
# FIXME-WT-9063 revisit the use of self.retry() throughout this file.
# Insert a value that is larger than the maximum leaf value.
for i in range(0, 10):
for retry in self.retry():
with retry.transaction(commit_timestamp=2):
cursor[self.make_key(i)] = value1
# Do 2 modifies.
for i in range(0, 10):
for retry in self.retry():
with retry.transaction(commit_timestamp=3):
cursor.set_key(self.make_key(i))
mods = [wiredtiger.Modify('B', 500, 1)]
self.assertEqual(cursor.modify(mods), 0)
for i in range(0, 10):
for retry in self.retry():
with retry.transaction(commit_timestamp=4):
cursor.set_key(self.make_key(i))
mods = [wiredtiger.Modify('C', 501, 1)]
self.assertEqual(cursor.modify(mods), 0)
# Insert more data to trigger eviction.
for i in range(10, 100000):
for retry in self.retry():
with retry.transaction(commit_timestamp=5):
cursor[self.make_key(i)] = value2
# Update the overflow values.
for i in range(0, 10):
for retry in self.retry():
with retry.transaction(commit_timestamp=5):
cursor[self.make_key(i)] = value2
# Do a checkpoint to move the overflow values to the history store but keep the current in memory disk image.
self.session.checkpoint()
# Search the first modifies.
for i in range(0, 10):
for retry in self.retry():
with retry.transaction(read_timestamp=3, rollback=True):
self.assertEqual(cursor[self.make_key(i)], value1 + "B")
def test_modify_insert_to_hs(self):
uri = "table:test_hs10"
uri2 = "table:test_hs10_otherdata"
create_params = 'value_format=S,key_format={}'.format(self.key_format)
value1 = 'a' * 1000
value2 = 'b' * 1000
self.session.create(uri, create_params)
session2 = self.setUpSessionOpen(self.conn)
session2.create(uri2, create_params)
cursor2 = session2.open_cursor(uri2)
# Insert a full value.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1))
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(1))
cursor = self.session.open_cursor(uri)
self.session.begin_transaction()
cursor[1] = value1
self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(2))
# Insert 3 modifies in separate transactions.
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('A', 1000, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(3))
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('B', 1001, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(4))
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('C', 1002, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' + self.timestamp_str(5))
self.session.checkpoint()
# Insert a whole bunch of data into the other table to force wiredtiger to evict data
# from the previous table.
for i in range(1, 10000):
cursor2[i] = value2
# Validate that we see the correct value at each of the timestamps.
self.session.begin_transaction('read_timestamp=' + self.timestamp_str(3))
cursor.set_key(1)
cursor.search()
self.assertEqual(cursor[1], value1 + 'A')
self.session.commit_transaction()
cursor2 = self.session.open_cursor(uri)
self.session.begin_transaction('read_timestamp=' + self.timestamp_str(4))
cursor2.set_key(1)
cursor2.search()
self.assertEqual(cursor2.get_value(), value1 + 'AB')
self.session.commit_transaction()
self.session.begin_transaction('read_timestamp=' + self.timestamp_str(5))
self.assertEqual(cursor[1], value1 + 'ABC')
self.session.commit_transaction()
def test_hs_multiple_modifies(self):
# FLCS doesn't support modify, so just skip over this test.
if self.value_format == '8t':
return
# Create a small table.
uri = "table:test_hs06"
create_params = 'key_format={},value_format={}'.format(
self.key_format, self.value_format)
self.session.create(uri, create_params)
value1 = 'a' * 500
value2 = 'b' * 500
# Load 1Mb of data.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1))
cursor = self.session.open_cursor(uri)
for i in range(1, self.nrows):
self.session.begin_transaction()
cursor[self.create_key(i)] = value1
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(2))
# Apply three sets of modifies.
# They specifically need to be in separate modify calls.
for i in range(1, 11):
self.session.begin_transaction()
cursor.set_key(self.create_key(i))
self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]),
0)
self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]),
0)
self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]),
0)
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(3))
expected = list(value1)
expected[100] = 'B'
expected[200] = 'C'
expected[300] = 'D'
expected = str().join(expected)
# Write a newer value on top.
for i in range(1, self.nrows):
self.session.begin_transaction()
cursor[self.create_key(i)] = value2
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(4))
# Go back and read. We should get the initial value with the 3 modifies applied on top.
self.session.begin_transaction('read_timestamp=' +
self.timestamp_str(3))
for i in range(1, 11):
self.assertEqual(cursor[self.create_key(i)], expected)
self.session.rollback_transaction()
def test_modify_abort(self):
ds = SimpleDataSet(self,
self.uri,
20,
key_format=self.keyfmt,
value_format=self.valuefmt)
ds.populate()
# Start a transaction.
self.session.begin_transaction("isolation=snapshot")
# Insert a new record.
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(30))
c.set_value(ds.value(30))
self.assertEquals(c.insert(), 0)
# Test that we can successfully modify our own record.
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
mods = self.fix_mods(mods)
self.assertEqual(c.modify(mods), 0)
# Test that another transaction cannot modify our uncommitted record.
xs = self.conn.open_session()
xc = xs.open_cursor(self.uri, None)
xs.begin_transaction("isolation=snapshot")
xc.set_key(ds.key(30))
xc.set_value(ds.value(30))
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
mods = self.fix_mods(mods)
xc.set_key(ds.key(30))
self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND)
xs.rollback_transaction()
# Rollback our transaction.
self.session.rollback_transaction()
# Test that we can't modify our aborted insert.
self.session.begin_transaction("isolation=snapshot")
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
mods = self.fix_mods(mods)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
self.session.rollback_transaction()
def modify_load(self, ds, single):
# For each test in the list:
# set the original value,
# apply modifications in order,
# confirm the final state
row = 10
c = self.session.open_cursor(self.uri, None)
for i in self.list:
c.set_key(ds.key(row))
c.set_value(self.make_value(i['o']))
self.assertEquals(c.update(), 0)
c.reset()
self.session.begin_transaction("isolation=snapshot")
c.set_key(ds.key(row))
mods = []
for j in i['mods']:
mod = wiredtiger.Modify(j[0], j[1], j[2])
mods.append(mod)
mods = self.fix_mods(mods)
self.assertEquals(c.modify(mods), 0)
self.session.commit_transaction()
c.reset()
c.set_key(ds.key(row))
self.assertEquals(c.search(), 0)
v = c.get_value()
expect = self.make_value(i['f'])
self.assertEquals(self.nulls_to_spaces(v), expect)
if not single:
row = row + 1
c.close()
def test_apply_modifies_on_onpage_tombstone(self):
self.session.create(self.uri, 'key_format=S,value_format=S')
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1))
cursor = self.session.open_cursor(self.uri)
value = 'a' * 500
for i in range(1, 10000):
self.session.begin_transaction()
cursor[str(i)] = value
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(2))
# Apply tombstones for every key.
for i in range(1, 10000):
self.session.begin_transaction()
cursor.set_key(str(i))
cursor.remove()
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(3))
self.session.checkpoint()
# Now try to apply a modify on top of the tombstone at timestamp 3.
for i in range(1, 10000):
self.session.begin_transaction()
cursor.set_key(str(i))
self.assertEqual(cursor.modify([wiredtiger.Modify('B', 0, 100)]),
wiredtiger.WT_NOTFOUND)
self.session.rollback_transaction()
# Check that the tombstone is visible.
for i in range(1, 10000):
cursor.set_key(str(i))
self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND)
def large_modifies(self, uri, value, ds, location, nbytes, nrows, prepare, commit_ts):
# Load a slight modification.
session = self.session
try:
cursor = session.open_cursor(uri)
session.begin_transaction()
for i in range(1, nrows + 1):
cursor.set_key(i)
mods = [wiredtiger.Modify(value, location, nbytes)]
self.assertEqual(cursor.modify(mods), 0)
if commit_ts == 0:
session.commit_transaction()
elif prepare:
session.prepare_transaction('prepare_timestamp=' + self.timestamp_str(commit_ts-1))
session.timestamp_transaction('commit_timestamp=' + self.timestamp_str(commit_ts))
session.timestamp_transaction('durable_timestamp=' + self.timestamp_str(commit_ts+1))
session.commit_transaction()
else:
session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts))
cursor.close()
except WiredTigerError as e:
rollback_str = wiredtiger_strerror(WT_ROLLBACK)
if rollback_str in str(e):
session.rollback_transaction()
raise(e)
def modify_load(self, ds, single):
# For each test in the list:
# set the original value,
# apply modifications in order,
# confirm the final state
row = 10
c = self.session.open_cursor(self.uri, None)
for i in self.list:
c.set_key(ds.key(row))
c.set_value(i['o'])
self.assertEquals(c.update(), 0)
c.reset()
c.set_key(ds.key(row))
mods = []
for j in i['mods']:
mod = wiredtiger.Modify(j[0], j[1], j[2])
mods.append(mod)
self.assertEquals(c.modify(mods), 0)
c.reset()
c.set_key(ds.key(row))
self.assertEquals(c.search(), 0)
v = c.get_value()
self.assertEquals(v.replace("\x00", " "), i['f'])
if not single:
row = row + 1
c.close()
def large_modifies(self, uri, value, ds, location, nbytes, nrows, prepare,
commit_ts):
# Load a slight modification.
session = self.session
cursor = session.open_cursor(uri)
session.begin_transaction()
for i in range(1, nrows + 1):
cursor.set_key(i)
mods = [wiredtiger.Modify(value, location, nbytes)]
self.assertEqual(cursor.modify(mods), 0)
if commit_ts == 0:
session.commit_transaction()
elif prepare:
session.prepare_transaction('prepare_timestamp=' +
timestamp_str(commit_ts - 1))
session.timestamp_transaction('commit_timestamp=' +
timestamp_str(commit_ts))
session.timestamp_transaction('durable_timestamp=' +
timestamp_str(commit_ts + 1))
session.commit_transaction()
else:
session.commit_transaction('commit_timestamp=' +
timestamp_str(commit_ts))
cursor.close()
def test_modify_many(self):
ds = SimpleDataSet(self,
self.uri,
20,
key_format=self.keyfmt,
value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
self.session.begin_transaction("isolation=snapshot")
c.set_key(ds.key(10))
orig = self.make_value('abcdefghijklmnopqrstuvwxyz')
c.set_value(orig)
self.assertEquals(c.update(), 0)
for i in range(0, 50000):
new = self.make_value("".join([random.choice(string.digits) \
for i in range(5)]))
orig = orig[:10] + new + orig[15:]
mods = []
mod = wiredtiger.Modify(new, 10, 5)
mods.append(mod)
mods = self.fix_mods(mods)
self.assertEquals(c.modify(mods), 0)
self.session.commit_transaction()
c.set_key(ds.key(10))
self.assertEquals(c.search(), 0)
self.assertEquals(c.get_value(), orig)
def large_modifies(self, uri, value, ds, location, nbytes, nrows, commit_ts):
# Load a slight modification.
session = self.session
cursor = session.open_cursor(uri)
session.begin_transaction()
for i in range(0, nrows):
cursor.set_key(i)
mods = [wiredtiger.Modify(value, location, nbytes)]
self.assertEqual(cursor.modify(mods), 0)
session.commit_transaction('commit_timestamp=' + self.timestamp_str(commit_ts))
cursor.close()
def fix_mods(self, mods):
if bytes != str and self.valuefmt == 'u':
# In Python3, bytes and strings are independent types, and
# the WiredTiger API needs bytes when the format calls for bytes.
newmods = []
for mod in mods:
# We need to check because we may converted some of the Modify
# records already.
if type(mod.data) == str:
newmods.append(wiredtiger.Modify(
self.make_value(mod.data), mod.offset, mod.size))
else:
newmods.append(mod)
mods = newmods
return mods
def test_modify_delete(self):
self.session.create(self.uri, 'key_format=S,value_format=u')
cursor = self.session.open_cursor(self.uri, None, None)
cursor['ABC'] = 'ABCDEFGH'
cursor.set_key('ABC')
cursor.remove()
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
cursor.set_key('ABC')
#self.assertEqual(cursor.modify(mods), wiredtiger.WT_NOTFOUND)
self.assertRaises(wiredtiger.WiredTigerError,
lambda: cursor.modify(mods))
def test_modify_delete(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
self.assertEquals(c.remove(), 0)
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(10))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
def large_modifies(self, session, uri, offset, ds, nrows, timestamp=False):
# Modify a large number of records, we'll hang if the history store table
# isn't doing its thing.
cursor = session.open_cursor(uri)
for i in range(1, 10000):
if timestamp == True:
session.begin_transaction()
cursor.set_key(ds.key(nrows + i))
mods = []
mod = wiredtiger.Modify('A', offset, 1)
mods.append(mod)
self.assertEqual(cursor.modify(mods), 0)
if timestamp == True:
session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1))
cursor.close()
def test_modify_delete(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
self.assertEquals(c.remove(), 0)
self.session.begin_transaction("isolation=snapshot")
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
mods = self.fix_mods(mods)
c.set_key(ds.key(10))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
self.session.commit_transaction()
def large_modifies(self, session, uri, offset, ds, nrows, timestamp=False):
cursor = session.open_cursor(uri)
for i in range(1, nrows):
# Unlike inserts and updates, modify operations do not implicitly start/commit a transaction.
# Hence, we begin/commit transaction manually.
session.begin_transaction()
cursor.set_key(ds.key(i))
mods = []
mod = wiredtiger.Modify('A', offset, 1)
mods.append(mod)
self.assertEqual(cursor.modify(mods), 0)
if timestamp == True:
session.commit_transaction('commit_timestamp=' +
self.timestamp_str(i + 1))
else:
session.commit_transaction()
cursor.close()
def test_insert_hs_full_update(self):
uri = 'table:test_hs28'
self.session.create(
uri, 'key_format={},value_format=S'.format(self.key_format))
value1 = "a"
value2 = "b"
cursor = self.session.open_cursor(uri)
# Insert a full value
self.session.begin_transaction()
cursor[1] = value1
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(2))
# Do a modify update
self.session.begin_transaction()
cursor.set_key(1)
mods = [wiredtiger.Modify('A', 0, 1)]
self.assertEqual(cursor.modify(mods), 0)
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(5))
# Commit a transaction with multiple updates on the same key
self.session.begin_transaction()
cursor[1] = value1
cursor[1] = value2
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(10))
# Move the updates to the history store
self.session.checkpoint()
stat_cursor = self.session.open_cursor('statistics:', None, None)
hs_full_update = stat_cursor[
wiredtiger.stat.conn.cache_hs_insert_full_update][2]
hs_reverse_modify = stat_cursor[
wiredtiger.stat.conn.cache_hs_insert_reverse_modify][2]
stat_cursor.close()
self.assertEqual(hs_full_update, 2)
self.assertEqual(hs_reverse_modify, 0)
def test_modify_many(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
orig = 'abcdefghijklmnopqrstuvwxyz'
c.set_value(orig)
self.assertEquals(c.update(), 0)
for i in range(0, 50000):
new = "".join([random.choice(string.digits) for i in xrange(5)])
orig = orig[:10] + new + orig[15:]
mods = []
mod = wiredtiger.Modify(new, 10, 5)
mods.append(mod)
self.assertEquals(c.modify(mods), 0)
c.set_key(ds.key(10))
self.assertEquals(c.search(), 0)
self.assertEquals(c.get_value(), orig)
def test_flcs(self):
uri = "table:test_flcs04"
nrows = 10
ds = SimpleDataSet(self,
uri,
nrows,
key_format='r',
value_format='6t',
config='leaf_page_max=4096')
ds.populate()
cursor = self.session.open_cursor(uri)
self.session.begin_transaction()
cursor.set_key(5)
mods = [wiredtiger.Modify('Q', 100, 1)]
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cursor.modify(mods),
"/WT_CURSOR.modify only supported for/")
self.session.rollback_transaction()
cursor.close()
def large_modifies(self, uri, value, ds, location, nbytes, nrows, prepare,
commit_ts):
# Load a slight modification.
session = self.session
try:
cursor = session.open_cursor(uri)
session.begin_transaction()
for i in range(1, nrows + 1):
cursor.set_key(i)
# FLCS doesn't support modify (for obvious reasons) so just update.
# Use the first character of the passed-in value.
if self.value_format == '8t':
cursor.set_value(bytes(value, encoding='utf-8')[0])
self.assertEqual(cursor.update(), 0)
else:
mods = [wiredtiger.Modify(value, location, nbytes)]
self.assertEqual(cursor.modify(mods), 0)
if commit_ts == 0:
session.commit_transaction()
elif prepare:
session.prepare_transaction('prepare_timestamp=' +
self.timestamp_str(commit_ts - 1))
session.timestamp_transaction('commit_timestamp=' +
self.timestamp_str(commit_ts))
session.timestamp_transaction('durable_timestamp=' +
self.timestamp_str(commit_ts +
1))
session.commit_transaction()
else:
session.commit_transaction('commit_timestamp=' +
self.timestamp_str(commit_ts))
cursor.close()
except WiredTigerError as e:
rollback_str = wiredtiger_strerror(WT_ROLLBACK)
if rollback_str in str(e):
session.rollback_transaction()
raise (e)
def large_modifies(self, session, uri, offset, ds, nrows, timestamp=False):
cursor = session.open_cursor(uri)
for i in range(1, nrows):
# Unlike inserts and updates, modify operations do not implicitly start/commit a transaction.
# Hence, we begin/commit transaction manually.
session.begin_transaction()
cursor.set_key(ds.key(i))
# FLCS doesn't allow modify (it doesn't make sense) so just update to 'j' then 'k'.
if self.value_format == '8t':
cursor.set_value(106 + offset)
self.assertEqual(cursor.update(), 0)
else:
mods = []
mod = wiredtiger.Modify('A', offset, 1)
mods.append(mod)
self.assertEqual(cursor.modify(mods), 0)
if timestamp == True:
session.commit_transaction('commit_timestamp=' +
self.timestamp_str(i + 1))
else:
session.commit_transaction()
cursor.close()
def test_modify_insert_to_hs(self):
uri = "table:test_hs08"
create_params = 'value_format=S,key_format={}'.format(self.key_format)
value1 = 'a' * 1000
self.session.create(uri, create_params)
# Insert a full value.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(1))
cursor = self.session.open_cursor(uri)
self.session.begin_transaction()
cursor[1] = value1
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(2))
# Insert 3 modifies in separate transactions.
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('A', 1000, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(3))
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('B', 1001, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(4))
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('C', 1002, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(5))
# Call checkpoint.
self.session.checkpoint('use_timestamp=true')
# Validate that we did write at least once to the history store.
hs_writes = self.get_stat(stat.conn.cache_write_hs)
squashed_write = self.get_stat(stat.conn.cache_hs_write_squash)
self.assertGreaterEqual(hs_writes, 1)
self.assertEqual(squashed_write, 0)
# Validate that we see the correct value at each of the timestamps.
self.session.begin_transaction('read_timestamp=' +
self.timestamp_str(3))
self.assertEqual(cursor[1], value1 + 'A')
self.session.commit_transaction()
self.session.begin_transaction('read_timestamp=' +
self.timestamp_str(4))
self.assertEqual(cursor[1], value1 + 'AB')
self.session.commit_transaction()
self.session.begin_transaction('read_timestamp=' +
self.timestamp_str(5))
self.assertEqual(cursor[1], value1 + 'ABC')
self.session.commit_transaction()
# Insert another two modifies. When we call checkpoint the first modify
# will get written to the data store as a full value and the second will
# be written to the data store as a reverse delta.
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('D', 1000, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(7))
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('E', 1001, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(8))
# Call checkpoint again.
self.session.checkpoint('use_timestamp=true')
# Validate that we wrote to the history store again.
hs_writes = self.get_stat(stat.conn.cache_write_hs)
squashed_write = self.get_stat(stat.conn.cache_hs_write_squash)
self.assertGreaterEqual(hs_writes, 2)
self.assertEqual(squashed_write, 0)
# Validate that we see the expected value on the modifies, this
# scenario tests the logic that will retrieve a full value for
# a modify previously inserted into the history store.
self.session.begin_transaction('read_timestamp=' +
self.timestamp_str(7))
self.assertEqual(cursor[1], value1 + 'DBC')
self.session.commit_transaction()
self.session.begin_transaction('read_timestamp=' +
self.timestamp_str(8))
self.assertEqual(cursor[1], value1 + 'DEC')
self.session.commit_transaction()
# Insert multiple modifies in the same transaction the first two should be squashed.
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0)
self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0)
self.assertEqual(cursor.modify([wiredtiger.Modify('H', 1004, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(9))
# Call checkpoint again.
self.session.checkpoint('use_timestamp=true')
# Validate that we squashed two modifies. Note we can't count the exact number
# we squashed, just that we did squash.
hs_writes = self.get_stat(stat.conn.cache_write_hs)
squashed_write = self.get_stat(stat.conn.cache_hs_write_squash)
self.assertGreaterEqual(hs_writes, 3)
self.assertEqual(squashed_write, 1)
# Insert multiple modifies in two different transactions so we should squash two.
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0)
self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(10))
self.session.begin_transaction()
cursor.set_key(1)
self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0)
self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0)
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(11))
# Call checkpoint again.
self.session.checkpoint('use_timestamp=true')
# Validate that we squashed two modifies. We also squashed a modify that was previously
# squashed hence the number actually goes up by three.
hs_writes = self.get_stat(stat.conn.cache_write_hs)
squashed_write = self.get_stat(stat.conn.cache_hs_write_squash)
self.assertGreaterEqual(hs_writes, 4)
self.assertEqual(squashed_write, 4)
# Insert multiple modifies in different transactions with different timestamps on each
# modify to guarantee we squash zero modifies.
self.session.begin_transaction()
cursor.set_key(1)
self.session.timestamp_transaction('commit_timestamp=' +
self.timestamp_str(12))
self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0)
self.session.timestamp_transaction('commit_timestamp=' +
self.timestamp_str(13))
self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0)
self.session.commit_transaction()
self.session.begin_transaction()
cursor.set_key(1)
self.session.timestamp_transaction('commit_timestamp=' +
self.timestamp_str(14))
self.assertEqual(cursor.modify([wiredtiger.Modify('F', 1002, 1)]), 0)
self.session.timestamp_transaction('commit_timestamp=' +
self.timestamp_str(15))
self.assertEqual(cursor.modify([wiredtiger.Modify('G', 1003, 1)]), 0)
self.session.commit_transaction()
# Call checkpoint again.
self.session.checkpoint('use_timestamp=true')
# Validate that we squashed zero modifies.
hs_writes = self.get_stat(stat.conn.cache_write_hs)
squashed_write = self.get_stat(stat.conn.cache_hs_write_squash)
self.assertGreaterEqual(hs_writes, 5)
self.assertEqual(squashed_write, 5)
def test_hs15(self):
uri = 'table:test_hs15'
self.session.create(
uri, 'key_format={},value_format=S'.format(self.key_format))
cursor = self.session.open_cursor(uri)
value1 = 'a' * 500
value2 = 'b' * 500
value3 = 'c' * 500
# Insert an update without timestamp
self.session.begin_transaction()
cursor[self.create_key(1)] = value1
self.session.commit_transaction()
# Insert a bunch of other contents to trigger eviction
for i in range(2, 1000):
self.session.begin_transaction()
cursor[self.create_key(i)] = value2
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(3))
# Do a modify and an update with timestamps
self.session.begin_transaction()
cursor.set_key(self.create_key(1))
mods = [wiredtiger.Modify('B', 100, 1)]
self.assertEqual(cursor.modify(mods), 0)
self.session.commit_transaction('commit_timestamp=' + timestamp_str(1))
self.session.begin_transaction()
cursor[self.create_key(1)] = value2
self.session.commit_transaction('commit_timestamp=' + timestamp_str(2))
# Make the modify with timestamp and the update without timestamp obsolete
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1))
# Do a checkpoint
self.session.checkpoint()
self.session.begin_transaction()
cursor[self.create_key(1)] = value3
self.session.commit_transaction('commit_timestamp=' + timestamp_str(3))
# Insert a bunch of other contents to trigger eviction
for i in range(2, 1000):
self.session.begin_transaction()
cursor[self.create_key(i)] = value3
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(3))
expected = list(value1)
expected[100] = 'B'
expected = str().join(expected)
self.session.begin_transaction('read_timestamp=' + timestamp_str(1))
self.assertEqual(cursor[self.create_key(1)], expected)
self.session.rollback_transaction()
self.session.begin_transaction('read_timestamp=' + timestamp_str(2))
self.assertEqual(cursor[self.create_key(1)], value2)
self.session.rollback_transaction()
self.session.begin_transaction('read_timestamp=' + timestamp_str(3))
self.assertEqual(cursor[self.create_key(1)], value3)
self.session.rollback_transaction()
def test_hs_rec_modify(self):
# Create a small table.
uri = "table:test_hs06"
create_params = 'key_format={},value_format=S'.format(self.key_format)
self.session.create(uri, create_params)
value1 = 'a' * 500
value2 = 'b' * 500
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1) +
',stable_timestamp=' + timestamp_str(1))
cursor = self.session.open_cursor(uri)
# Base update.
for i in range(1, 10000):
self.session.begin_transaction()
cursor[self.create_key(i)] = value1
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(2))
# Apply three sets of modifies.
for i in range(1, 11):
self.session.begin_transaction()
cursor.set_key(self.create_key(i))
self.assertEqual(cursor.modify([wiredtiger.Modify('B', 100, 1)]),
0)
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(3))
for i in range(1, 11):
self.session.begin_transaction()
cursor.set_key(self.create_key(i))
self.assertEqual(cursor.modify([wiredtiger.Modify('C', 200, 1)]),
0)
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(4))
# This is the one we want to be selected by the checkpoint.
for i in range(1, 11):
self.session.begin_transaction()
cursor.set_key(self.create_key(i))
self.assertEqual(cursor.modify([wiredtiger.Modify('D', 300, 1)]),
0)
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(5))
# Apply another update and evict the pages with the modifies out of cache.
for i in range(1, 10000):
self.session.begin_transaction()
cursor[self.create_key(i)] = value2
self.session.commit_transaction('commit_timestamp=' +
timestamp_str(6))
# Checkpoint such that the modifies will be selected. When we grab it from the history
# store, we'll need to unflatten it before using it for reconciliation.
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(5))
self.session.checkpoint()
expected = list(value1)
expected[100] = 'B'
expected[200] = 'C'
expected[300] = 'D'
expected = str().join(expected)
# Check that the correct value is visible after checkpoint.
self.session.begin_transaction('read_timestamp=' + timestamp_str(5))
for i in range(1, 11):
self.assertEqual(cursor[self.create_key(i)], expected)
self.session.rollback_transaction()