class test_cursor_comparison(wttest.WiredTigerTestCase):
name = 'test_compare'
types = [('file', dict(type='file:', lsm=False, dataset=SimpleDataSet)),
('lsm', dict(type='table:', lsm=True, dataset=ComplexLSMDataSet)),
('table', dict(type='table:', lsm=False, dataset=ComplexDataSet))]
keyfmt = [('integer', dict(keyfmt='i')), ('recno', dict(keyfmt='r')),
('string', dict(keyfmt='S'))]
# Skip record number keys with LSM.
scenarios = filter_scenarios(
make_scenarios(types, keyfmt),
lambda name, d: not (d['lsm'] and d['keyfmt'] == 'r'))
def test_cursor_comparison(self):
uri = self.type + 'compare'
uriX = self.type + 'compareX'
# Build the object.
ds = self.dataset(self, uri, 100, key_format=self.keyfmt)
dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt)
ds.populate()
dsX.populate()
if self.type == 'file:':
ix0_0 = None
ix0_1 = None
ix1_0 = None
ixX_0 = None
else:
ix0_0 = self.session.open_cursor(ds.index_name(0), None)
ix0_1 = self.session.open_cursor(ds.index_name(0), None)
ix1_0 = self.session.open_cursor(ds.index_name(1), None)
ixX_0 = self.session.open_cursor(dsX.index_name(0), None)
ix0_0.next()
ix0_1.next()
ix1_0.next()
ixX_0.next()
c1 = self.session.open_cursor(uri, None)
c2 = self.session.open_cursor(uri, None)
# Confirm failure unless the keys are set.
msg = '/requires key be set/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c1.compare(c2), msg)
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c2.compare(c1), msg)
# Test cursors before they're positioned.
c1.set_key(ds.key(10))
c2.set_key(ds.key(20))
self.assertGreater(c2.compare(c1), 0)
self.assertLess(c1.compare(c2), 0)
c2.set_key(ds.key(10))
self.assertEqual(c1.compare(c2), 0)
self.assertEqual(c2.compare(c1), 0)
# Confirm failure for different objects.
cX = self.session.open_cursor(uriX, None)
cX.set_key(dsX.key(10))
msg = '/must reference the same object/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cX.compare(c1), msg)
msg = '/wt_cursor.* is None/'
self.assertRaisesHavingMessage(RuntimeError, lambda: cX.compare(None),
msg)
if ix0_0 != None:
self.assertEqual(ix0_0.compare(ix0_1), 0)
ix0_1.reset()
ix0_1.prev()
self.assertLess(ix0_0.compare(ix0_1), 0)
self.assertGreater(ix0_1.compare(ix0_0), 0)
# Main table vs. index not allowed
msg = '/must reference the same object/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c1.compare(ix0_0), msg)
# Two unrelated indices not allowed
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: ixX_0.compare(ix0_0), msg)
# Two different indices from same table not allowed
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: ix0_0.compare(ix1_0), msg)
# Test cursors after they're positioned (shouldn't matter for compare).
c1.set_key(ds.key(10))
self.assertEqual(c1.search(), 0)
c2.set_key(ds.key(20))
self.assertEqual(c2.search(), 0)
self.assertGreater(c2.compare(c1), 0)
self.assertLess(c1.compare(c2), 0)
c2.set_key(ds.key(10))
self.assertEqual(c2.search(), 0)
self.assertEqual(c1.compare(c2), 0)
self.assertEqual(c2.compare(c1), 0)
# Confirm failure for different objects.
cX = self.session.open_cursor(uriX, None)
cX.set_key(dsX.key(10))
self.assertEqual(cX.search(), 0)
msg = '/must reference the same object/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cX.compare(c1), msg)
def test_cursor_equality(self):
uri = self.type + 'equality'
uriX = self.type + 'compareX'
# Build the object.
ds = self.dataset(self, uri, 100, key_format=self.keyfmt)
dsX = self.dataset(self, uriX, 100, key_format=self.keyfmt)
ds.populate()
dsX.populate()
if self.type == 'file:':
ix0_0 = None
ix0_1 = None
ix1_0 = None
ixX_0 = None
else:
ix0_0 = self.session.open_cursor(ds.index_name(0), None)
ix0_1 = self.session.open_cursor(ds.index_name(0), None)
ix1_0 = self.session.open_cursor(ds.index_name(1), None)
ixX_0 = self.session.open_cursor(dsX.index_name(0), None)
ix0_0.next()
ix0_1.next()
ix1_0.next()
ixX_0.next()
c1 = self.session.open_cursor(uri, None)
c2 = self.session.open_cursor(uri, None)
# Confirm failure unless the keys are set.
msg = '/requires key be set/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c1.equals(c2), msg)
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c2.equals(c1), msg)
# Test cursors before they're positioned.
c1.set_key(ds.key(10))
c2.set_key(ds.key(20))
self.assertFalse(c1.equals(c2))
self.assertFalse(c2.equals(c1))
c2.set_key(ds.key(10))
self.assertTrue(c1.equals(c2))
self.assertTrue(c2.equals(c1))
# Confirm failure for different objects.
cX = self.session.open_cursor(uriX, None)
cX.set_key(dsX.key(10))
msg = '/must reference the same object/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cX.equals(c1), msg)
msg = '/wt_cursor.* is None/'
self.assertRaisesHavingMessage(RuntimeError, lambda: cX.equals(None),
msg)
if ix0_0 != None:
self.assertTrue(ix0_0.equals(ix0_1))
ix0_1.reset()
ix0_1.prev()
self.assertFalse(ix0_0.equals(ix0_1))
# Main table vs. index not allowed
msg = '/must reference the same object/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: c1.equals(ix0_0), msg)
# Two unrelated indices not allowed
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: ixX_0.equals(ix0_0), msg)
# Two different indices from same table not allowed
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: ix0_0.equals(ix1_0), msg)
# Test cursors after they're positioned (internally, it's a different
# search path if keys are positioned in the tree).
c1.set_key(ds.key(10))
self.assertEqual(c1.search(), 0)
c2.set_key(ds.key(20))
self.assertEqual(c2.search(), 0)
self.assertFalse(c1.equals(c2))
self.assertFalse(c2.equals(c1))
c2.set_key(ds.key(10))
self.assertEqual(c2.search(), 0)
self.assertTrue(c1.equals(c2))
self.assertTrue(c2.equals(c1))
# Confirm failure for different objects.
cX = self.session.open_cursor(uriX, None)
cX.set_key(dsX.key(10))
self.assertEqual(cX.search(), 0)
msg = '/must reference the same object/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: cX.equals(c1), msg)
class test_cursor12(wttest.WiredTigerTestCase):
keyfmt = [
('recno', dict(keyfmt='r')),
('string', dict(keyfmt='S')),
]
valuefmt = [
('item', dict(valuefmt='u')),
('string', dict(valuefmt='S')),
]
types = [
('file', dict(uri='file:modify')),
('lsm', dict(uri='lsm:modify')),
('table', dict(uri='table:modify')),
]
# Skip record number keys with LSM.
scenarios = filter_scenarios(make_scenarios(types, keyfmt, valuefmt),
lambda name, d: not ('lsm' in d['uri'] and d['keyfmt'] == 'r'))
# List with original value, final value, and modifications to get
# there.
list = [
{
'o' : 'ABCDEFGH', # no operation
'f' : 'ABCDEFGH',
'mods' : [['', 0, 0]]
},{
'o' : 'ABCDEFGH', # no operation with offset
'f' : 'ABCDEFGH',
'mods' : [['', 4, 0]]
},{
'o' : 'ABCDEFGH', # rewrite beginning
'f' : '--CDEFGH',
'mods' : [['--', 0, 2]]
},{
'o' : 'ABCDEFGH', # rewrite end
'f' : 'ABCDEF--',
'mods' : [['--', 6, 2]]
},{
'o' : 'ABCDEFGH', # append
'f' : 'ABCDEFGH--',
'mods' : [['--', 8, 2]]
},{
'o' : 'ABCDEFGH', # append with gap
'f' : 'ABCDEFGH --',
'mods' : [['--', 10, 2]]
},{
'o' : 'ABCDEFGH', # multiple replacements
'f' : 'A-C-E-G-',
'mods' : [['-', 1, 1], ['-', 3, 1], ['-', 5, 1], ['-', 7, 1]]
},{
'o' : 'ABCDEFGH', # multiple overlapping replacements
'f' : 'A-CDEFGH',
'mods' : [['+', 1, 1], ['+', 1, 1], ['+', 1, 1], ['-', 1, 1]]
},{
'o' : 'ABCDEFGH', # multiple overlapping gap replacements
'f' : 'ABCDEFGH --',
'mods' : [['+', 10, 1], ['+', 10, 1], ['+', 10, 1], ['--', 10, 2]]
},{
'o' : 'ABCDEFGH', # shrink beginning
'f' : '--EFGH',
'mods' : [['--', 0, 4]]
},{
'o' : 'ABCDEFGH', # shrink middle
'f' : 'AB--GH',
'mods' : [['--', 2, 4]]
},{
'o' : 'ABCDEFGH', # shrink end
'f' : 'ABCD--',
'mods' : [['--', 4, 4]]
},{
'o' : 'ABCDEFGH', # grow beginning
'f' : '--ABCDEFGH',
'mods' : [['--', 0, 0]]
},{
'o' : 'ABCDEFGH', # grow middle
'f' : 'ABCD--EFGH',
'mods' : [['--', 4, 0]]
},{
'o' : 'ABCDEFGH', # grow end
'f' : 'ABCDEFGH--',
'mods' : [['--', 8, 0]]
},{
'o' : 'ABCDEFGH', # discard beginning
'f' : 'EFGH',
'mods' : [['', 0, 4]]
},{
'o' : 'ABCDEFGH', # discard middle
'f' : 'ABGH',
'mods' : [['', 2, 4]]
},{
'o' : 'ABCDEFGH', # discard end
'f' : 'ABCD',
'mods' : [['', 4, 4]]
},{
'o' : 'ABCDEFGH', # discard everything
'f' : '',
'mods' : [['', 0, 8]]
},{
'o' : 'ABCDEFGH', # overlap the end and append
'f' : 'ABCDEF--XX',
'mods' : [['--XX', 6, 2]]
},{
'o' : 'ABCDEFGH', # overlap the end with incorrect size
'f' : 'ABCDEFG01234567',
'mods' : [['01234567', 7, 2000]]
},{ # many updates
'o' : '-ABCDEFGHIJKLMNOPQRSTUVWXYZ-',
'f' : '-eeeeeeeeeeeeeeeeeeeeeeeeee-',
'mods' : [['a', 1, 1], ['a', 2, 1], ['a', 3, 1], ['a', 4, 1],
['a', 5, 1], ['a', 6, 1], ['a', 7, 1], ['a', 8, 1],
['a', 9, 1], ['a', 10, 1], ['a', 11, 1], ['a', 12, 1],
['a', 13, 1], ['a', 14, 1], ['a', 15, 1], ['a', 16, 1],
['a', 17, 1], ['a', 18, 1], ['a', 19, 1], ['a', 20, 1],
['a', 21, 1], ['a', 22, 1], ['a', 23, 1], ['a', 24, 1],
['a', 25, 1], ['a', 26, 1],
['b', 1, 1], ['b', 2, 1], ['b', 3, 1], ['b', 4, 1],
['b', 5, 1], ['b', 6, 1], ['b', 7, 1], ['b', 8, 1],
['b', 9, 1], ['b', 10, 1], ['b', 11, 1], ['b', 12, 1],
['b', 13, 1], ['b', 14, 1], ['b', 15, 1], ['b', 16, 1],
['b', 17, 1], ['b', 18, 1], ['b', 19, 1], ['b', 20, 1],
['b', 21, 1], ['b', 22, 1], ['b', 23, 1], ['b', 24, 1],
['b', 25, 1], ['b', 26, 1],
['c', 1, 1], ['c', 2, 1], ['c', 3, 1], ['c', 4, 1],
['c', 5, 1], ['c', 6, 1], ['c', 7, 1], ['c', 8, 1],
['c', 9, 1], ['c', 10, 1], ['c', 11, 1], ['c', 12, 1],
['c', 13, 1], ['c', 14, 1], ['c', 15, 1], ['c', 16, 1],
['c', 17, 1], ['c', 18, 1], ['c', 19, 1], ['c', 20, 1],
['c', 21, 1], ['c', 22, 1], ['c', 23, 1], ['c', 24, 1],
['c', 25, 1], ['c', 26, 1],
['d', 1, 1], ['d', 2, 1], ['d', 3, 1], ['d', 4, 1],
['d', 5, 1], ['d', 6, 1], ['d', 7, 1], ['d', 8, 1],
['d', 9, 1], ['d', 10, 1], ['d', 11, 1], ['d', 12, 1],
['d', 13, 1], ['d', 14, 1], ['d', 15, 1], ['d', 16, 1],
['d', 17, 1], ['d', 18, 1], ['d', 19, 1], ['d', 20, 1],
['d', 21, 1], ['d', 22, 1], ['d', 23, 1], ['d', 24, 1],
['d', 25, 1], ['d', 26, 1],
['e', 1, 1], ['e', 2, 1], ['e', 3, 1], ['e', 4, 1],
['e', 5, 1], ['e', 6, 1], ['e', 7, 1], ['e', 8, 1],
['e', 9, 1], ['e', 10, 1], ['e', 11, 1], ['e', 12, 1],
['e', 13, 1], ['e', 14, 1], ['e', 15, 1], ['e', 16, 1],
['e', 17, 1], ['e', 18, 1], ['e', 19, 1], ['e', 20, 1],
['e', 21, 1], ['e', 22, 1], ['e', 23, 1], ['e', 24, 1],
['e', 25, 1], ['e', 26, 1]]
}
]
# Create a set of modified records and verify in-memory reads.
def modify_load(self, ds, single):
# For each test in the list:
# set the original value,
# apply modifications in order,
# confirm the final state
row = 10
c = self.session.open_cursor(self.uri, None)
for i in self.list:
c.set_key(ds.key(row))
c.set_value(i['o'])
self.assertEquals(c.update(), 0)
c.reset()
c.set_key(ds.key(row))
mods = []
for j in i['mods']:
mod = wiredtiger.Modify(j[0], j[1], j[2])
mods.append(mod)
self.assertEquals(c.modify(mods), 0)
c.reset()
c.set_key(ds.key(row))
self.assertEquals(c.search(), 0)
v = c.get_value()
self.assertEquals(v.replace("\x00", " "), i['f'])
if not single:
row = row + 1
c.close()
# Confirm the modified records are correct.
def modify_confirm(self, ds, single):
# For each test in the list:
# confirm the final state is there.
row = 10
c = self.session.open_cursor(self.uri, None)
for i in self.list:
c.set_key(ds.key(row))
self.assertEquals(c.search(), 0)
v = c.get_value()
self.assertEquals(v.replace("\x00", " "), i['f'])
if not single:
row = row + 1
c.close()
# Smoke-test the modify API, operating on a group of records.
def test_modify_smoke(self):
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.modify_load(ds, False)
# Smoke-test the modify API, operating on a single record
def test_modify_smoke_single(self):
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.modify_load(ds, True)
# Smoke-test the modify API, closing and re-opening the database.
def test_modify_smoke_reopen(self):
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.modify_load(ds, False)
# Flush to disk, forcing reconciliation.
self.reopen_conn()
self.modify_confirm(ds, False)
# Smoke-test the modify API, recovering the database.
def test_modify_smoke_recover(self):
# Close the original database.
self.conn.close()
# Open a new database with logging configured.
self.conn_config = \
'log=(enabled=true),transaction_sync=(method=dsync,enabled)'
self.conn = self.setUpConnectionOpen(".")
self.session = self.setUpSessionOpen(self.conn)
# Populate a database, and checkpoint it so it exists after recovery.
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.session.checkpoint()
self.modify_load(ds, False)
# Crash and recover in a new directory.
newdir = 'RESTART'
copy_wiredtiger_home('.', newdir)
self.conn.close()
self.conn = self.setUpConnectionOpen(newdir)
self.session = self.setUpSessionOpen(self.conn)
self.session.verify(self.uri)
self.modify_confirm(ds, False)
# Check that we can perform a large number of modifications to a record.
def test_modify_many(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
orig = 'abcdefghijklmnopqrstuvwxyz'
c.set_value(orig)
self.assertEquals(c.update(), 0)
for i in range(0, 50000):
new = "".join([random.choice(string.digits) for i in xrange(5)])
orig = orig[:10] + new + orig[15:]
mods = []
mod = wiredtiger.Modify(new, 10, 5)
mods.append(mod)
self.assertEquals(c.modify(mods), 0)
c.set_key(ds.key(10))
self.assertEquals(c.search(), 0)
self.assertEquals(c.get_value(), orig)
# Check that modify returns not-found after a delete.
def test_modify_delete(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
self.assertEquals(c.remove(), 0)
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(10))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
# Check that modify returns not-found when an insert is not yet committed
# and after it's aborted.
def test_modify_abort(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
# Start a transaction.
self.session.begin_transaction()
# Insert a new record.
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(30))
c.set_value(ds.value(30))
self.assertEquals(c.insert(), 0)
# Test that we can successfully modify our own record.
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), 0)
# Test that another transaction cannot modify our uncommitted record.
xs = self.conn.open_session()
xc = xs.open_cursor(self.uri, None)
xc.set_key(ds.key(30))
xc.set_value(ds.value(30))
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
xc.set_key(ds.key(30))
self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND)
# Rollback our transaction.
self.session.rollback_transaction()
# Test that we can't modify our aborted insert.
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
class test_cursor12(wttest.WiredTigerTestCase):
keyfmt = [
('recno', dict(keyfmt='r')),
('string', dict(keyfmt='S')),
]
valuefmt = [
('item', dict(valuefmt='u')),
('string', dict(valuefmt='S')),
]
types = [
('file', dict(uri='file:modify')),
('lsm', dict(uri='lsm:modify')),
('table', dict(uri='table:modify')),
]
# Skip record number keys with LSM.
scenarios = filter_scenarios(make_scenarios(types, keyfmt, valuefmt),
lambda name, d: not ('lsm' in d['uri'] and d['keyfmt'] == 'r'))
# List with original value, final value, and modifications to get
# there.
list = [
{
'o' : 'ABCDEFGH', # no operation
'f' : 'ABCDEFGH',
'mods' : [['', 0, 0]]
},{
'o' : 'ABCDEFGH', # no operation with offset
'f' : 'ABCDEFGH',
'mods' : [['', 4, 0]]
},{
'o' : 'ABCDEFGH', # rewrite beginning
'f' : '--CDEFGH',
'mods' : [['--', 0, 2]]
},{
'o' : 'ABCDEFGH', # rewrite end
'f' : 'ABCDEF--',
'mods' : [['--', 6, 2]]
},{
'o' : 'ABCDEFGH', # append
'f' : 'ABCDEFGH--',
'mods' : [['--', 8, 2]]
},{
'o' : 'ABCDEFGH', # append with gap
'f' : 'ABCDEFGH --',
'mods' : [['--', 10, 2]]
},{
'o' : 'ABCDEFGH', # multiple replacements
'f' : 'A-C-E-G-',
'mods' : [['-', 1, 1], ['-', 3, 1], ['-', 5, 1], ['-', 7, 1]]
},{
'o' : 'ABCDEFGH', # multiple overlapping replacements
'f' : 'A-CDEFGH',
'mods' : [['+', 1, 1], ['+', 1, 1], ['+', 1, 1], ['-', 1, 1]]
},{
'o' : 'ABCDEFGH', # multiple overlapping gap replacements
'f' : 'ABCDEFGH --',
'mods' : [['+', 10, 1], ['+', 10, 1], ['+', 10, 1], ['--', 10, 2]]
},{
'o' : 'ABCDEFGH', # shrink beginning
'f' : '--EFGH',
'mods' : [['--', 0, 4]]
},{
'o' : 'ABCDEFGH', # shrink middle
'f' : 'AB--GH',
'mods' : [['--', 2, 4]]
},{
'o' : 'ABCDEFGH', # shrink end
'f' : 'ABCD--',
'mods' : [['--', 4, 4]]
},{
'o' : 'ABCDEFGH', # grow beginning
'f' : '--ABCDEFGH',
'mods' : [['--', 0, 0]]
},{
'o' : 'ABCDEFGH', # grow middle
'f' : 'ABCD--EFGH',
'mods' : [['--', 4, 0]]
},{
'o' : 'ABCDEFGH', # grow end
'f' : 'ABCDEFGH--',
'mods' : [['--', 8, 0]]
},{
'o' : 'ABCDEFGH', # discard beginning
'f' : 'EFGH',
'mods' : [['', 0, 4]]
},{
'o' : 'ABCDEFGH', # discard middle
'f' : 'ABGH',
'mods' : [['', 2, 4]]
},{
'o' : 'ABCDEFGH', # discard end
'f' : 'ABCD',
'mods' : [['', 4, 4]]
},{
'o' : 'ABCDEFGH', # discard everything
'f' : '',
'mods' : [['', 0, 8]]
},{
'o' : 'ABCDEFGH', # overlap the end and append
'f' : 'ABCDEF--XX',
'mods' : [['--XX', 6, 2]]
},{
'o' : 'ABCDEFGH', # overlap the end with incorrect size
'f' : 'ABCDEFG01234567',
'mods' : [['01234567', 7, 2000]]
},{ # many updates
'o' : '-ABCDEFGHIJKLMNOPQRSTUVWXYZ-',
'f' : '-eeeeeeeeeeeeeeeeeeeeeeeeee-',
'mods' : [['a', 1, 1], ['a', 2, 1], ['a', 3, 1], ['a', 4, 1],
['a', 5, 1], ['a', 6, 1], ['a', 7, 1], ['a', 8, 1],
['a', 9, 1], ['a', 10, 1], ['a', 11, 1], ['a', 12, 1],
['a', 13, 1], ['a', 14, 1], ['a', 15, 1], ['a', 16, 1],
['a', 17, 1], ['a', 18, 1], ['a', 19, 1], ['a', 20, 1],
['a', 21, 1], ['a', 22, 1], ['a', 23, 1], ['a', 24, 1],
['a', 25, 1], ['a', 26, 1],
['b', 1, 1], ['b', 2, 1], ['b', 3, 1], ['b', 4, 1],
['b', 5, 1], ['b', 6, 1], ['b', 7, 1], ['b', 8, 1],
['b', 9, 1], ['b', 10, 1], ['b', 11, 1], ['b', 12, 1],
['b', 13, 1], ['b', 14, 1], ['b', 15, 1], ['b', 16, 1],
['b', 17, 1], ['b', 18, 1], ['b', 19, 1], ['b', 20, 1],
['b', 21, 1], ['b', 22, 1], ['b', 23, 1], ['b', 24, 1],
['b', 25, 1], ['b', 26, 1],
['c', 1, 1], ['c', 2, 1], ['c', 3, 1], ['c', 4, 1],
['c', 5, 1], ['c', 6, 1], ['c', 7, 1], ['c', 8, 1],
['c', 9, 1], ['c', 10, 1], ['c', 11, 1], ['c', 12, 1],
['c', 13, 1], ['c', 14, 1], ['c', 15, 1], ['c', 16, 1],
['c', 17, 1], ['c', 18, 1], ['c', 19, 1], ['c', 20, 1],
['c', 21, 1], ['c', 22, 1], ['c', 23, 1], ['c', 24, 1],
['c', 25, 1], ['c', 26, 1],
['d', 1, 1], ['d', 2, 1], ['d', 3, 1], ['d', 4, 1],
['d', 5, 1], ['d', 6, 1], ['d', 7, 1], ['d', 8, 1],
['d', 9, 1], ['d', 10, 1], ['d', 11, 1], ['d', 12, 1],
['d', 13, 1], ['d', 14, 1], ['d', 15, 1], ['d', 16, 1],
['d', 17, 1], ['d', 18, 1], ['d', 19, 1], ['d', 20, 1],
['d', 21, 1], ['d', 22, 1], ['d', 23, 1], ['d', 24, 1],
['d', 25, 1], ['d', 26, 1],
['e', 1, 1], ['e', 2, 1], ['e', 3, 1], ['e', 4, 1],
['e', 5, 1], ['e', 6, 1], ['e', 7, 1], ['e', 8, 1],
['e', 9, 1], ['e', 10, 1], ['e', 11, 1], ['e', 12, 1],
['e', 13, 1], ['e', 14, 1], ['e', 15, 1], ['e', 16, 1],
['e', 17, 1], ['e', 18, 1], ['e', 19, 1], ['e', 20, 1],
['e', 21, 1], ['e', 22, 1], ['e', 23, 1], ['e', 24, 1],
['e', 25, 1], ['e', 26, 1]]
}
]
def setUp(self):
if sys.version_info[0] >= 3 and self.valuefmt == 'u':
# Python3 distinguishes bytes from strings
self.nullbyte = b'\x00'
self.spacebyte = b' '
else:
self.nullbyte = '\x00'
self.spacebyte = ' '
super(test_cursor12, self).setUp()
# Convert a string to the correct type for the value.
def make_value(self, s):
if self.valuefmt == 'u':
return bytes(s.encode())
else:
return s
def fix_mods(self, mods):
if bytes != str and self.valuefmt == 'u':
# In Python3, bytes and strings are independent types, and
# the WiredTiger API needs bytes when the format calls for bytes.
newmods = []
for mod in mods:
# We need to check because we may converted some of the Modify
# records already.
if type(mod.data) == str:
newmods.append(wiredtiger.Modify(
self.make_value(mod.data), mod.offset, mod.size))
else:
newmods.append(mod)
mods = newmods
return mods
# Create a set of modified records and verify in-memory reads.
def modify_load(self, ds, single):
# For each test in the list:
# set the original value,
# apply modifications in order,
# confirm the final state
row = 10
c = self.session.open_cursor(self.uri, None)
for i in self.list:
c.set_key(ds.key(row))
c.set_value(self.make_value(i['o']))
self.assertEquals(c.update(), 0)
c.reset()
self.session.begin_transaction("isolation=snapshot")
c.set_key(ds.key(row))
mods = []
for j in i['mods']:
mod = wiredtiger.Modify(j[0], j[1], j[2])
mods.append(mod)
mods = self.fix_mods(mods)
self.assertEquals(c.modify(mods), 0)
self.session.commit_transaction()
c.reset()
c.set_key(ds.key(row))
self.assertEquals(c.search(), 0)
v = c.get_value()
expect = self.make_value(i['f'])
self.assertEquals(v.replace(self.nullbyte, self.spacebyte), expect)
if not single:
row = row + 1
c.close()
# Confirm the modified records are correct.
def modify_confirm(self, ds, single):
# For each test in the list:
# confirm the final state is there.
row = 10
c = self.session.open_cursor(self.uri, None)
for i in self.list:
c.set_key(ds.key(row))
self.assertEquals(c.search(), 0)
v = c.get_value()
expect = self.make_value(i['f'])
self.assertEquals(v.replace(self.nullbyte, self.spacebyte), expect)
if not single:
row = row + 1
c.close()
# Smoke-test the modify API, anything other than an explicit transaction
# in snapshot isolation fails.
def test_modify_txn_api(self):
ds = SimpleDataSet(self, self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
msg = '/not supported/'
self.session.begin_transaction()
mods = []
mods.append(wiredtiger.Modify('-', 1, 1))
self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c.modify(mods), msg)
self.session.rollback_transaction()
self.session.begin_transaction("isolation=read-uncommitted")
mods = []
mods.append(wiredtiger.Modify('-', 1, 1))
self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c.modify(mods), msg)
self.session.rollback_transaction()
self.session.begin_transaction("isolation=read-committed")
mods = []
mods.append(wiredtiger.Modify('-', 1, 1))
self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: c.modify(mods), msg)
self.session.rollback_transaction()
# Smoke-test the modify API, operating on a group of records.
def test_modify_smoke(self):
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.modify_load(ds, False)
# Smoke-test the modify API, operating on a single record
def test_modify_smoke_single(self):
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.modify_load(ds, True)
# Smoke-test the modify API, closing and re-opening the database.
def test_modify_smoke_reopen(self):
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.modify_load(ds, False)
# Flush to disk, forcing reconciliation.
self.reopen_conn()
self.modify_confirm(ds, False)
# Smoke-test the modify API, recovering the database.
def test_modify_smoke_recover(self):
# Close the original database.
self.conn.close()
# Open a new database with logging configured.
self.conn_config = \
'log=(enabled=true),transaction_sync=(method=dsync,enabled)'
self.conn = self.setUpConnectionOpen(".")
self.session = self.setUpSessionOpen(self.conn)
# Populate a database, and checkpoint it so it exists after recovery.
ds = SimpleDataSet(self,
self.uri, 100, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
self.session.checkpoint()
self.modify_load(ds, False)
# Crash and recover in a new directory.
newdir = 'RESTART'
copy_wiredtiger_home('.', newdir)
self.conn.close()
self.conn = self.setUpConnectionOpen(newdir)
self.session = self.setUpSessionOpen(self.conn)
self.session.verify(self.uri)
self.modify_confirm(ds, False)
# Check that we can perform a large number of modifications to a record.
def test_modify_many(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
self.session.begin_transaction("isolation=snapshot")
c.set_key(ds.key(10))
orig = self.make_value('abcdefghijklmnopqrstuvwxyz')
c.set_value(orig)
self.assertEquals(c.update(), 0)
for i in range(0, 50000):
new = self.make_value("".join([random.choice(string.digits) \
for i in range(5)]))
orig = orig[:10] + new + orig[15:]
mods = []
mod = wiredtiger.Modify(new, 10, 5)
mods.append(mod)
mods = self.fix_mods(mods)
self.assertEquals(c.modify(mods), 0)
self.session.commit_transaction()
c.set_key(ds.key(10))
self.assertEquals(c.search(), 0)
self.assertEquals(c.get_value(), orig)
# Check that modify returns not-found after a delete.
def test_modify_delete(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(10))
self.assertEquals(c.remove(), 0)
self.session.begin_transaction("isolation=snapshot")
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
mods = self.fix_mods(mods)
c.set_key(ds.key(10))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
self.session.commit_transaction()
# Check that modify returns not-found when an insert is not yet committed
# and after it's aborted.
def test_modify_abort(self):
ds = SimpleDataSet(self,
self.uri, 20, key_format=self.keyfmt, value_format=self.valuefmt)
ds.populate()
# Start a transaction.
self.session.begin_transaction("isolation=snapshot")
# Insert a new record.
c = self.session.open_cursor(self.uri, None)
c.set_key(ds.key(30))
c.set_value(ds.value(30))
self.assertEquals(c.insert(), 0)
# Test that we can successfully modify our own record.
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
c.set_key(ds.key(30))
mods = self.fix_mods(mods)
self.assertEqual(c.modify(mods), 0)
# Test that another transaction cannot modify our uncommitted record.
xs = self.conn.open_session()
xc = xs.open_cursor(self.uri, None)
xs.begin_transaction("isolation=snapshot")
xc.set_key(ds.key(30))
xc.set_value(ds.value(30))
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
mods = self.fix_mods(mods)
xc.set_key(ds.key(30))
self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND)
xs.rollback_transaction()
# Rollback our transaction.
self.session.rollback_transaction()
# Test that we can't modify our aborted insert.
self.session.begin_transaction("isolation=snapshot")
mods = []
mod = wiredtiger.Modify('ABCD', 3, 3)
mods.append(mod)
mods = self.fix_mods(mods)
c.set_key(ds.key(30))
self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND)
self.session.rollback_transaction()
class test_overwrite(wttest.WiredTigerTestCase):
name = 'overwrite'
keyfmt = [
('integer', dict(keyfmt='i')),
('recno', dict(keyfmt='r')),
('string', dict(keyfmt='S')),
]
types = [
('file', dict(uri='file:', lsm=False, ds=SimpleDataSet)),
('lsm', dict(uri='lsm:', lsm=True, ds=SimpleDataSet)),
('table-complex', dict(uri='table:', lsm=False, ds=ComplexDataSet)),
('table-complex-lsm', dict(uri='table:',
lsm=True,
ds=ComplexLSMDataSet)),
('table-index', dict(uri='table:', lsm=False, ds=SimpleIndexDataSet)),
('table-simple', dict(uri='table:', lsm=False, ds=SimpleDataSet)),
('table-simple-lsm', dict(uri='table:', lsm=True,
ds=SimpleLSMDataSet)),
]
# Skip record number keys with LSM.
scenarios = filter_scenarios(
make_scenarios(types, keyfmt),
lambda name, d: not (d['lsm'] and d['keyfmt'] == 'r'))
# Confirm a cursor configured with/without overwrite correctly handles
# non-existent records during insert, remove and update operations.
def test_overwrite_insert(self):
uri = self.uri + self.name
ds = self.ds(self, uri, 100, key_format=self.keyfmt)
ds.populate()
# Insert of an existing record with overwrite off fails.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
cursor.set_key(ds.key(5))
cursor.set_value(ds.value(1000))
self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert())
# One additional test for the insert method: duplicate the cursor with
# overwrite configured and then the insert should succeed. This test
# is only for the insert method because the remove and update method
# failure modes are for non-existent records, and you cannot duplicate
# cursor pointing to non-existent records.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
cursor.set_key(ds.key(5))
dupc = self.session.open_cursor(None, cursor, "overwrite=true")
dupc.set_value(ds.value(1001))
self.assertEquals(dupc.insert(), 0)
# Insert of an existing record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
cursor.set_key(ds.key(6))
cursor.set_value(ds.value(1002))
self.assertEquals(cursor.insert(), 0)
# Insert of a non-existent record with overwrite off succeeds.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
cursor.set_key(ds.key(200))
cursor.set_value(ds.value(1003))
self.assertEquals(cursor.insert(), 0)
# Insert of a non-existent record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
cursor.set_key(ds.key(201))
cursor.set_value(ds.value(1004))
self.assertEquals(cursor.insert(), 0)
def test_overwrite_remove(self):
uri = self.uri + self.name
ds = self.ds(self, uri, 100, key_format=self.keyfmt)
ds.populate()
# Remove of an existing record with overwrite off succeeds.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
cursor.set_key(ds.key(5))
self.assertEquals(cursor.remove(), 0)
# Remove of an existing record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
cursor.set_key(ds.key(6))
self.assertEquals(cursor.remove(), 0)
# Remove of a non-existent record with overwrite off fails.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
cursor.set_key(ds.key(200))
self.assertEquals(cursor.remove(), wiredtiger.WT_NOTFOUND)
# Remove of a non-existent record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
cursor.set_key(ds.key(201))
self.assertEquals(cursor.remove(), 0)
def test_overwrite_update(self):
uri = self.uri + self.name
ds = self.ds(self, uri, 100, key_format=self.keyfmt)
ds.populate()
# Update of an existing record with overwrite off succeeds.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
cursor.set_key(ds.key(5))
cursor.set_value(ds.value(1005))
self.assertEquals(cursor.update(), 0)
# Update of an existing record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
cursor.set_key(ds.key(6))
cursor.set_value(ds.value(1006))
self.assertEquals(cursor.update(), 0)
# Update of a non-existent record with overwrite off fails.
cursor = self.session.open_cursor(uri, None, "overwrite=false")
cursor.set_key(ds.key(200))
cursor.set_value(ds.value(1007))
self.assertEquals(cursor.update(), wiredtiger.WT_NOTFOUND)
# Update of a non-existent record with overwrite on succeeds.
cursor = self.session.open_cursor(uri, None)
cursor.set_key(ds.key(201))
cursor.set_value(ds.value(1008))
self.assertEquals(cursor.update(), 0)
class test_rollback_to_stable25(wttest.WiredTigerTestCase):
conn_config = 'in_memory=false'
write_10_values = [
('10u', dict(write_10='u')),
('10h', dict(write_10='h')),
('10f', dict(write_10='f')),
('10m', dict(write_10='m')),
('10l', dict(write_10='l')),
]
type_10_values = [
('nil', dict(type_10=None)),
('upd', dict(type_10='upd')),
]
write_20_values = [
('20u', dict(write_20='u')),
('20h', dict(write_20='h')),
('20f', dict(write_20='f')),
('20m', dict(write_20='m')),
('20l', dict(write_20='l')),
]
type_20_values = [
('nil', dict(type_20=None)),
('upd', dict(type_20='upd')),
('del', dict(type_20='del')),
]
write_30_values = [
('30u', dict(write_30='u')),
('30h', dict(write_30='h')),
('30f', dict(write_30='f')),
('30m', dict(write_30='m')),
('30l', dict(write_30='l')),
]
type_30_values = [
('nil', dict(type_30=None)),
('upd', dict(type_30='upd')),
('del', dict(type_30='del')),
]
evict_time_values = [
('chk10', dict(evict_time=10)),
('chk20', dict(evict_time=20)),
('chk30', dict(evict_time=30)),
]
rollback_time_values = [
('roll15', dict(rollback_time=15)),
('roll25', dict(rollback_time=25)),
]
def is_meaningful(name, vals):
# The last write at evict time should be uniform, to get an RLE cell.
if vals['evict_time'] == 10 and vals['write_10'] != 'u':
return False
if vals['evict_time'] == 20 and vals['write_20'] != 'u':
return False
if vals['evict_time'] == 30 and vals['write_30'] != 'u':
return False
# If the type is nil, the value must be uniform.
if vals['type_10'] is None and vals['write_10'] != 'u':
return False
if vals['type_20'] is None and vals['write_20'] != 'u':
return False
if vals['type_30'] is None and vals['write_30'] != 'u':
return False
# Similarly, delete and heterogeneous doesn't make sense.
if vals['type_10'] == 'del' and vals['write_10'] == 'h':
return False
if vals['type_20'] == 'del' and vals['write_20'] == 'h':
return False
if vals['type_20'] == 'del' and vals['write_30'] == 'h':
return False
# Both 10 and 20 shouldn't be nil. That's equivalent to 10 and 30 being nil.
if vals['type_10'] is None and vals['type_20'] is None:
return False
# Avoid cases that delete nonexistent values.
def deletes_nonexistent():
present = {}
for k in range(2, 2 + my_rle_size):
present[k] = False
def adjust(ty, write):
if ty is None:
return
for k in keys_of_write(write):
if ty == 'upd':
present[k] = True
elif ty == 'del':
if present[k]:
present[k] = False
else:
raise KeyError
adjust(vals['type_10'], vals['write_10'])
adjust(vals['type_20'], vals['write_20'])
adjust(vals['type_30'], vals['write_30'])
try:
deletes_nonexistent()
except KeyError:
return False
return True
scenarios = filter_scenarios(
make_scenarios(write_10_values, type_10_values, write_20_values,
type_20_values, write_30_values, type_30_values,
evict_time_values, rollback_time_values), is_meaningful)
value_z = "zzzzz" * 10
def writes(self, uri, s, expected, ty, write, value, ts):
if ty is None:
# do nothing at all
return
cursor = s.open_cursor(uri)
s.begin_transaction()
for k in keys_of_write(write):
if ty == 'upd':
myval = value + str(k) if write == 'h' else value
cursor[k] = myval
expected[k] = myval
else:
cursor.set_key(k)
cursor.remove()
del expected[k]
s.commit_transaction('commit_timestamp=' + self.timestamp_str(ts))
cursor.close()
def evict(self, uri, s):
# Evict the page to force reconciliation.
evict_cursor = s.open_cursor(uri, None, "debug=(release_evict)")
s.begin_transaction()
# Search the key to evict it. Use both bookends.
v = evict_cursor[1]
self.assertEqual(v, self.value_z)
v = evict_cursor[2 + my_rle_size]
self.assertEqual(v, self.value_z)
self.assertEqual(evict_cursor.reset(), 0)
s.rollback_transaction()
evict_cursor.close()
def check(self, uri, s, ts, expected):
cursor = s.open_cursor(uri)
s.begin_transaction('read_timestamp=' + self.timestamp_str(ts))
# endpoints should still be in place
self.assertEqual(cursor[1], self.value_z)
self.assertEqual(cursor[2 + my_rle_size], self.value_z)
for k in range(2, 2 + my_rle_size):
if k in expected:
self.assertEqual(cursor[k], expected[k])
else:
cursor.set_key(k)
r = cursor.search()
self.assertEqual(r, wiredtiger.WT_NOTFOUND)
s.rollback_transaction()
cursor.close()
def test_rollback_to_stable25(self):
# Create a table without logging.
uri = "table:rollback_to_stable25"
self.session.create(uri, 'key_format=r,value_format=S')
# Pin oldest timestamp to 5.
self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(5))
# Start stable timestamp at 5.
self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(5))
value_a = "aaaaa" * 10
value_b = "bbbbb" * 10
value_c = "ccccc" * 10
s = self.conn.open_session()
# Write the endpoints at time 5.
cursor = s.open_cursor(uri)
s.begin_transaction()
cursor[1] = self.value_z
cursor[2 + my_rle_size] = self.value_z
s.commit_transaction('commit_timestamp=' + self.timestamp_str(5))
self.evict(uri, s)
cursor.close()
# Do writes at time 10.
expected = {}
self.writes(uri, s, expected, self.type_10, self.write_10, value_a, 10)
expected10 = expected.copy()
# Evict at time 10 if requested.
if self.evict_time == 10:
self.evict(uri, s)
# Do more writes at time 20.
self.writes(uri, s, expected, self.type_20, self.write_20, value_b, 20)
expected20 = expected.copy()
# Evict at time 20 if requested.
if self.evict_time == 20:
self.evict(uri, s)
# Do still more writes at time 30.
self.writes(uri, s, expected, self.type_30, self.write_30, value_c, 30)
expected30 = expected.copy()
# Evict at time 30 if requested.
if self.evict_time == 30:
self.evict(uri, s)
# Now roll back.
self.conn.set_timestamp('stable_timestamp=' +
self.timestamp_str(self.rollback_time))
self.conn.rollback_to_stable()
if self.rollback_time < 20:
expected20 = expected10
expected30 = expected10
elif self.rollback_time < 30:
expected30 = expected20
# Now make sure we see what we expect.
self.check(uri, s, 10, expected10)
self.check(uri, s, 20, expected20)
self.check(uri, s, 30, expected30)
