class test_lsm01(wttest.WiredTigerTestCase):
K = 1024
M = 1024 * K
G = 1024 * M
uri = "lsm:test_lsm01"
chunk_size_scenarios = wtscenario.quick_scenarios('s_chunk_size',
[1 * M, 20 * M, None],
[0.6, 0.6, 0.6])
merge_max_scenarios = wtscenario.quick_scenarios('s_merge_max',
[2, 10, 20, None], None)
bloom_scenarios = wtscenario.quick_scenarios('s_bloom',
[True, False, None], None)
bloom_bit_scenarios = wtscenario.quick_scenarios('s_bloom_bit_count',
[2, 8, 20, None], None)
bloom_hash_scenarios = wtscenario.quick_scenarios('s_bloom_hash_count',
[2, 10, 20, None], None)
# Occasionally add a lot of records, so that merges (and bloom) happen.
record_count_scenarios = wtscenario.quick_scenarios(
'nrecs', [10, 10000], [0.9, 0.1])
config_vars = [
'chunk_size', 'merge_max', 'bloom', 'bloom_bit_count',
'bloom_hash_count'
]
all_scenarios = wtscenario.multiply_scenarios(
'_', chunk_size_scenarios, merge_max_scenarios, bloom_scenarios,
bloom_bit_scenarios, bloom_hash_scenarios, record_count_scenarios)
scenarios = wtscenario.prune_scenarios(all_scenarios, 500)
scenarios = wtscenario.number_scenarios(scenarios)
# Test drop of an object.
def test_lsm(self):
args = 'key_format=S'
args += ',lsm=(' # Start the LSM configuration options.
# add names to args, e.g. args += ',session_max=30'
for var in self.config_vars:
value = getattr(self, 's_' + var)
if value != None:
if var == 'verbose':
value = '[' + str(value) + ']'
if value == True:
value = 'true'
if value == False:
value = 'false'
args += ',' + var + '=' + str(value)
args += ')' # Close the LSM configuration option group
self.verbose(
3, 'Test LSM with config: ' + args + ' count: ' + str(self.nrecs))
simple_populate(self, self.uri, args, self.nrecs)
class test_config03(test_base03.test_base03):
K = 1024
M = 1024 * K
G = 1024 * M
cache_size_scenarios = wtscenario.quick_scenarios(
's_cache_size', [1 * M, 20 * M, 100 * M, 1 * G, None],
[0.6, 0.6, 0.6, 0.6, 0.6])
create_scenarios = wtscenario.quick_scenarios('s_create',
[True, False, None],
[1.0, 0.2, 0.3])
error_prefix_scenarios = wtscenario.quick_scenarios(
's_error_prefix', [None, "errpfx:"], [1.0, 0.2])
# eviction_target < eviction_trigger -- checked later
eviction_target_scenarios = wtscenario.quick_scenarios(
's_eviction_target', [10, 40, 85, 98], None)
eviction_trigger_scenarios = wtscenario.quick_scenarios(
's_eviction_trigger', [50, 90, 95, 99], None)
hazard_max_scenarios = wtscenario.quick_scenarios('s_hazard_max',
[15, 50, 500],
[0.4, 0.8, 0.8])
multiprocess_scenarios = wtscenario.quick_scenarios(
's_multiprocess', [True, False], [1.0, 1.0])
session_max_scenarios = wtscenario.quick_scenarios('s_session_max',
[3, 30, 300], None)
transactional_scenarios = wtscenario.quick_scenarios(
's_transactional', [True, False], [0.2, 1.0])
# Note: we are not using any truly verbose scenarios until we have
# a way to redirect verbose output to a file in Python.
#
#verbose_scenarios = wtscenario.quick_scenarios('s_verbose',
# ['block', 'evict,evictserver', 'fileops,hazard,mutex',
# 'read,readserver,reconcile,salvage','verify,write',''], None)
verbose_scenarios = wtscenario.quick_scenarios('s_verbose', [None], None)
config_vars = [
'cache_size', 'create', 'error_prefix', 'eviction_target',
'eviction_trigger', 'hazard_max', 'multiprocess', 'session_max',
'verbose'
]
all_scenarios = wtscenario.multiply_scenarios(
'_', cache_size_scenarios, create_scenarios, error_prefix_scenarios,
eviction_target_scenarios, eviction_trigger_scenarios,
hazard_max_scenarios, multiprocess_scenarios, session_max_scenarios,
transactional_scenarios, verbose_scenarios)
scenarios = wtscenario.prune_scenarios(all_scenarios, 1000)
scenarios = wtscenario.number_scenarios(scenarios)
#wttest.WiredTigerTestCase.printVerbose(2, 'test_config03: running ' + \
# str(len(scenarios)) + ' of ' + \
# str(len(all_scenarios)) + ' possible scenarios')
def setUpConnectionOpen(self, dir):
args = ''
# add names to args, e.g. args += ',session_max=30'
for var in self.config_vars:
value = getattr(self, 's_' + var)
if value != None:
if var == 'verbose':
value = '[' + str(value) + ']'
if value == True:
value = 'true'
if value == False:
value = 'false'
args += ',' + var + '=' + str(value)
args += ','
self.pr('wiredtiger_open with args: ' + args)
expect_fail = False
successargs = args
if self.s_create == False:
successargs = successargs.replace(',create=false,', ',create,')
expect_fail = True
fail_msg = '/(No such file or directory|The system cannot find the file specified)/'
elif self.s_create == None:
successargs = successargs + 'create=true,'
expect_fail = True
fail_msg = '/(No such file or directory|The system cannot find the file specified)/'
if self.s_eviction_target >= self.s_eviction_trigger:
# construct args that guarantee that target < trigger
# we know that trigger >= 1
repfrom = ',eviction_target=' + str(self.s_eviction_target)
repto = ',eviction_target=' + str(self.s_eviction_trigger - 1)
successargs = successargs.replace(repfrom, repto)
if not expect_fail:
expect_fail = True
fail_msg = \
'/eviction target must be lower than the eviction trigger/'
if expect_fail:
self.verbose(3, 'wiredtiger_open (should fail) with args: ' + args)
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: wiredtiger.wiredtiger_open(dir, args), fail_msg)
args = successargs
self.verbose(3, 'wiredtiger_open with args: ' + args)
conn = wiredtiger.wiredtiger_open(dir, args)
self.pr( ` conn `)
return conn
class test_txn02(wttest.WiredTigerTestCase, suite_subprocess):
logmax = "100K"
tablename = 'test_txn02'
uri = 'table:' + tablename
archive_list = ['true', 'false']
conn_list = ['reopen', 'stay_open']
sync_list = [
'(method=dsync,enabled)', '(method=fsync,enabled)',
'(method=none,enabled)', '(enabled=false)'
]
types = [
('row',
dict(tabletype='row', create_params='key_format=i,value_format=i')),
('var',
dict(tabletype='var', create_params='key_format=r,value_format=i')),
('fix',
dict(tabletype='fix', create_params='key_format=r,value_format=8t')),
]
op1s = [
('i4', dict(op1=('insert', 4))),
('r1', dict(op1=('remove', 1))),
('u10', dict(op1=('update', 10))),
]
op2s = [
('i6', dict(op2=('insert', 6))),
('r4', dict(op2=('remove', 4))),
('u4', dict(op2=('update', 4))),
]
op3s = [
('i12', dict(op3=('insert', 12))),
('r4', dict(op3=('remove', 4))),
('u4', dict(op3=('update', 4))),
]
op4s = [
('i14', dict(op4=('insert', 14))),
('r12', dict(op4=('remove', 12))),
('u12', dict(op4=('update', 12))),
]
txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))]
txn2s = [('t2c', dict(txn2='commit')), ('t2r', dict(txn2='rollback'))]
txn3s = [('t3c', dict(txn3='commit')), ('t3r', dict(txn3='rollback'))]
txn4s = [('t4c', dict(txn4='commit')), ('t4r', dict(txn4='rollback'))]
all_scenarios = multiply_scenarios('.', types, op1s, txn1s, op2s, txn2s,
op3s, txn3s, op4s, txn4s)
# This test generates thousands of potential scenarios.
# For default runs, we'll use a small subset of them, for
# long runs (when --long is set) we'll set a much larger limit.
scenarios = number_scenarios(prune_scenarios(all_scenarios, 20, 5000))
# Each check_log() call takes a second, so we don't call it for
# every scenario, we'll limit it to the value of checklog_calls.
checklog_calls = 100 if wttest.islongtest() else 2
checklog_mod = (len(scenarios) / checklog_calls + 1)
# scenarios = number_scenarios(multiply_scenarios('.', types,
# op1s, txn1s, op2s, txn2s, op3s, txn3s, op4s, txn4s)) [:3]
# Overrides WiredTigerTestCase
def setUpConnectionOpen(self, dir):
self.home = dir
# Cycle through the different transaction_sync values in a
# deterministic manner.
self.txn_sync = self.sync_list[self.scenario_number %
len(self.sync_list)]
#
# We don't want to run zero fill with only the same settings, such
# as archive or sync, which are an even number of options.
#
freq = 3
zerofill = 'false'
if self.scenario_number % freq == 0:
zerofill = 'true'
self.backup_dir = os.path.join(self.home, "WT_BACKUP")
conn_params = \
'log=(archive=false,enabled,file_max=%s),' % self.logmax + \
'log=(zero_fill=%s),' % zerofill + \
'create,error_prefix="%s: ",' % self.shortid() + \
'transaction_sync="%s",' % self.txn_sync
# print "Creating conn at '%s' with config '%s'" % (dir, conn_params)
conn = wiredtiger_open(dir, conn_params)
self.pr( ` conn `)
self.session2 = conn.open_session()
return conn
# Check that a cursor (optionally started in a new transaction), sees the
# expected values.
def check(self, session, txn_config, expected):
if txn_config:
session.begin_transaction(txn_config)
c = session.open_cursor(self.uri, None)
actual = dict((k, v) for k, v in c if v != 0)
# Search for the expected items as well as iterating
for k, v in expected.iteritems():
self.assertEqual(c[k], v)
c.close()
if txn_config:
session.commit_transaction()
self.assertEqual(actual, expected)
# Check the state of the system with respect to the current cursor and
# different isolation levels.
def check_all(self, current, committed):
# Transactions see their own changes.
# Read-uncommitted transactions see all changes.
# Snapshot and read-committed transactions should not see changes.
self.check(self.session, None, current)
self.check(self.session2, "isolation=snapshot", committed)
self.check(self.session2, "isolation=read-committed", committed)
self.check(self.session2, "isolation=read-uncommitted", current)
# Opening a clone of the database home directory should run
# recovery and see the committed results.
self.backup(self.backup_dir)
backup_conn_params = 'log=(enabled,file_max=%s)' % self.logmax
backup_conn = wiredtiger_open(self.backup_dir, backup_conn_params)
try:
self.check(backup_conn.open_session(), None, committed)
finally:
backup_conn.close()
def check_log(self, committed):
self.backup(self.backup_dir)
#
# Open and close the backup connection a few times to force
# repeated recovery and log archiving even if later recoveries
# are essentially no-ops. Confirm that the backup contains
# the committed operations after recovery.
#
# Cycle through the different archive values in a
# deterministic manner.
self.archive = self.archive_list[self.scenario_number %
len(self.archive_list)]
backup_conn_params = \
'log=(enabled,file_max=%s,archive=%s)' % (self.logmax, self.archive)
orig_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*")
endcount = 2
count = 0
while count < endcount:
backup_conn = wiredtiger_open(self.backup_dir, backup_conn_params)
try:
self.check(backup_conn.open_session(), None, committed)
finally:
# Sleep long enough so that the archive thread is guaranteed
# to run before we close the connection.
time.sleep(1.0)
backup_conn.close()
count += 1
#
# Check logs after repeated openings. The first log should
# have been archived if configured. Subsequent openings would not
# archive because no checkpoint is written due to no modifications.
#
cur_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*")
for o in orig_logs:
if self.archive == 'true':
self.assertEqual(False, o in cur_logs)
else:
self.assertEqual(True, o in cur_logs)
#
# Run printlog and make sure it exits with zero status.
# Printlog should not run recovery nor advance the logs. Make sure
# it does not.
#
self.runWt(['-h', self.backup_dir, 'printlog'],
outfilename='printlog.out')
pr_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*")
self.assertEqual(cur_logs, pr_logs)
def test_ops(self):
# print "Creating %s with config '%s'" % (self.uri, self.create_params)
self.session.create(self.uri, self.create_params)
# Set up the table with entries for 1, 2, 10 and 11.
# We use the overwrite config so insert can update as needed.
c = self.session.open_cursor(self.uri, None, 'overwrite')
c[1] = c[2] = c[10] = c[11] = 1
current = {1: 1, 2: 1, 10: 1, 11: 1}
committed = current.copy()
reopen = self.conn_list[self.scenario_number % len(self.conn_list)]
ops = (self.op1, self.op2, self.op3, self.op4)
txns = (self.txn1, self.txn2, self.txn3, self.txn4)
# for ok, txn in zip(ops, txns):
# print ', '.join('%s(%d)[%s]' % (ok[0], ok[1], txn)
for i, ot in enumerate(zip(ops, txns)):
ok, txn = ot
op, k = ok
# Close and reopen the connection and cursor.
if reopen == 'reopen':
self.reopen_conn()
c = self.session.open_cursor(self.uri, None, 'overwrite')
self.session.begin_transaction(
(self.scenario_number % 2) and 'sync' or None)
# Test multiple operations per transaction by always
# doing the same operation on key k + 1.
k1 = k + 1
# print '%d: %s(%d)[%s]' % (i, ok[0], ok[1], txn)
if op == 'insert' or op == 'update':
c[k] = c[k1] = i + 2
current[k] = current[k1] = i + 2
elif op == 'remove':
c.set_key(k)
c.remove()
c.set_key(k1)
c.remove()
if k in current:
del current[k]
if k1 in current:
del current[k1]
# print current
# Check the state after each operation.
self.check_all(current, committed)
if txn == 'commit':
committed = current.copy()
self.session.commit_transaction()
elif txn == 'rollback':
current = committed.copy()
self.session.rollback_transaction()
# Check the state after each commit/rollback.
self.check_all(current, committed)
# check_log() is slow, we don't run it on every scenario.
if self.scenario_number % test_txn02.checklog_mod == 0:
self.check_log(committed)
class test_txn09(wttest.WiredTigerTestCase, suite_subprocess):
tablename = 'test_txn09'
uri = 'table:' + tablename
log_enabled = True
types = [
('row',
dict(tabletype='row', create_params='key_format=i,value_format=i')),
('var',
dict(tabletype='var', create_params='key_format=r,value_format=i')),
('fix',
dict(tabletype='fix', create_params='key_format=r,value_format=8t')),
]
op1s = [
('i4', dict(op1=('insert', 4))),
('r1', dict(op1=('remove', 1))),
('u10', dict(op1=('update', 10))),
]
op2s = [
('i6', dict(op2=('insert', 6))),
('r4', dict(op2=('remove', 4))),
('u4', dict(op2=('update', 4))),
]
op3s = [
('i12', dict(op3=('insert', 12))),
('r4', dict(op3=('remove', 4))),
('u4', dict(op3=('update', 4))),
]
op4s = [
('i14', dict(op4=('insert', 14))),
('r12', dict(op4=('remove', 12))),
('u12', dict(op4=('update', 12))),
]
txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))]
txn2s = [('t2c', dict(txn2='commit')), ('t2r', dict(txn2='rollback'))]
txn3s = [('t3c', dict(txn3='commit')), ('t3r', dict(txn3='rollback'))]
txn4s = [('t4c', dict(txn4='commit')), ('t4r', dict(txn4='rollback'))]
all_scenarios = multiply_scenarios('.', types, op1s, txn1s, op2s, txn2s,
op3s, txn3s, op4s, txn4s)
# This test generates thousands of potential scenarios.
# For default runs, we'll use a small subset of them, for
# long runs (when --long is set) we'll set a much larger limit.
scenarios = number_scenarios(prune_scenarios(all_scenarios, 20, 5000))
# Overrides WiredTigerTestCase
def setUpConnectionOpen(self, dir):
self.home = dir
conn_params = \
'create,error_prefix="%s: ",' % self.shortid() + \
'log=(archive=false,enabled=%s),' % int(self.log_enabled) + \
'transaction_sync=(enabled=false),'
# print "Opening conn at '%s' with config '%s'" % (dir, conn_params)
conn = wiredtiger_open(dir, conn_params)
self.pr( ` conn `)
self.session2 = conn.open_session()
return conn
# Check that a cursor (optionally started in a new transaction), sees the
# expected values.
def check(self, session, txn_config, expected):
if txn_config:
session.begin_transaction(txn_config)
c = session.open_cursor(self.uri, None)
actual = dict((k, v) for k, v in c if v != 0)
# Search for the expected items as well as iterating
for k, v in expected.iteritems():
self.assertEqual(c[k], v)
c.close()
if txn_config:
session.commit_transaction()
self.assertEqual(actual, expected)
# Check the state of the system with respect to the current cursor and
# different isolation levels.
def check_all(self, current, committed):
# Transactions see their own changes.
# Read-uncommitted transactions see all changes.
# Snapshot and read-committed transactions should not see changes.
self.check(self.session, None, current)
self.check(self.session2, "isolation=snapshot", committed)
self.check(self.session2, "isolation=read-committed", committed)
self.check(self.session2, "isolation=read-uncommitted", current)
def test_ops(self):
# print "Creating %s with config '%s'" % (self.uri, self.create_params)
self.session.create(self.uri, self.create_params)
# Set up the table with entries for 1, 2, 10 and 11.
# We use the overwrite config so insert can update as needed.
c = self.session.open_cursor(self.uri, None, 'overwrite')
c[1] = c[2] = c[10] = c[11] = 1
current = {1: 1, 2: 1, 10: 1, 11: 1}
committed = current.copy()
ops = (self.op1, self.op2, self.op3, self.op4)
txns = (self.txn1, self.txn2, self.txn3, self.txn4)
# for ok, txn in zip(ops, txns):
# print ', '.join('%s(%d)[%s]' % (ok[0], ok[1], txn)
for i, ot in enumerate(zip(ops, txns)):
ok, txn = ot
op, k = ok
# Close and reopen the connection and cursor, toggling the log
self.log_enabled = not self.log_enabled
self.reopen_conn()
c = self.session.open_cursor(self.uri, None, 'overwrite')
self.session.begin_transaction(
(self.scenario_number % 2) and 'sync' or None)
# Test multiple operations per transaction by always
# doing the same operation on key k + 1.
k1 = k + 1
# print '%d: %s(%d)[%s]' % (i, ok[0], ok[1], txn)
if op == 'insert' or op == 'update':
c[k] = c[k1] = i + 2
current[k] = current[k1] = i + 2
elif op == 'remove':
del c[k]
del c[k1]
if k in current:
del current[k]
if k1 in current:
del current[k1]
# print current
# Check the state after each operation.
self.check_all(current, committed)
if txn == 'commit':
committed = current.copy()
self.session.commit_transaction()
elif txn == 'rollback':
current = committed.copy()
self.session.rollback_transaction()
# Check the state after each commit/rollback.
self.check_all(current, committed)
class test_schema03(wttest.WiredTigerTestCase):
"""
Test schemas - a 'predictably random' assortment of columns,
column groups and indices are created within tables, and are
created in various orders as much as the API allows. On some runs
the connection will be closed and reopened at a particular point
to test that the schemas (and data) are saved and read correctly.
The test is run multiple times, using scenarios.
The test always follows these steps:
- table: create tables
- colgroup0: create (some) colgroups
- index0: create (some) indices
- colgroup1: create (more) colgroups
- index1: create (more) indices
- populate0: populate 1st time
- index2: create (more) indices
- populate1: populate 2nd time (more key/values)
- check: check key/values
The variations represented by scenarios are:
- how many tables to create
- how many colgroups to create at each step (may be 0)
- how many indices to create at each step (may be 0)
- between each step, whether to close/reopen the connection
"""
################################################################
# These three variables can be altered to help generate
# and pare down failing test cases.
# Set to true to get python test program fragment on stdout,
# used by show_python() below.
SHOW_PYTHON = False
# When SHOW_PYTHON is set, we print an enormous amount of output.
# To only print for a given scenario, set this
SHOW_PYTHON_ONLY_SCEN = None # could be e.g. [2] or [0,1]
# To print verbosely for only a given table, set this
SHOW_PYTHON_ONLY_TABLE = None # could be e.g. [2] or [0,1]
################################################################
# Set whenever we are working with a table
current_table = None
nentries = 50
# We need to have a large number of open files available
# to run this test. We probably don't need quite this many,
# but boost it up to this limit anyway.
OPEN_FILE_LIMIT = 1000
restart_scenarios = [('table', dict(s_restart=['table'],P=0.3)),
('colgroup0', dict(s_restart=['colgroup0'],P=0.3)),
('index0', dict(s_restart=['index0'],P=0.3)),
('colgroup1', dict(s_restart=['colgroup1'],P=0.3)),
('index1', dict(s_restart=['index1'],P=0.3)),
('populate0', dict(s_restart=['populate0'],P=0.3)),
('index2', dict(s_restart=['index2'],P=0.3)),
('populate1', dict(s_restart=['populate1'],P=0.3)),
('ipop', dict(s_restart=['index0','populate0'],P=0.3)),
('all', dict(s_restart=['table','colgroup0','index0','colgroup1','index1','populate0','index2','populate1'],P=1.0))]
ntable_scenarios = wtscenario.quick_scenarios('s_ntable',
[1,2,5,8], [1.0,0.4,0.5,0.5])
ncolgroup_scenarios = wtscenario.quick_scenarios('s_colgroup',
[[1,0],[0,1],[2,4],[8,5]], [1.0,0.2,0.3,1.0])
nindex_scenarios = wtscenario.quick_scenarios('s_index',
[[1,1,1],[3,2,1],[5,1,3]], [1.0,0.5,1.0])
idx_args_scenarios = wtscenario.quick_scenarios('s_index_args',
['', ',type=file', ',type=lsm'], [0.5, 0.3, 0.2])
table_args_scenarios = wtscenario.quick_scenarios('s_extra_table_args',
['', ',type=file', ',type=lsm'], [0.5, 0.3, 0.2])
all_scenarios = wtscenario.multiply_scenarios('_', restart_scenarios, ntable_scenarios, ncolgroup_scenarios, nindex_scenarios, idx_args_scenarios, table_args_scenarios)
# Prune the scenarios according to the probabilities given above.
scenarios = wtscenario.prune_scenarios(all_scenarios, 30)
scenarios = wtscenario.number_scenarios(scenarios)
# Note: the set can be reduced here for debugging, e.g.
# scenarios = scenarios[40:44]
# or
# scenarios = [ scenarios[0], scenarios[30], scenarios[40] ]
#wttest.WiredTigerTestCase.printVerbose(2, 'test_schema03: running ' + \
# str(len(scenarios)) + ' of ' + \
# str(len(all_scenarios)) + ' possible scenarios')
# This test requires a large number of open files.
# Increase our resource limits before we start
def setUp(self):
if os.name == "nt":
self.skipTest('Unix specific test skipped on Windows')
self.origFileLimit = resource.getrlimit(resource.RLIMIT_NOFILE)
newlimit = (self.OPEN_FILE_LIMIT, self.origFileLimit[1])
if newlimit[0] > newlimit[1]:
self.skipTest('Require %d open files, only %d available' % newlimit)
resource.setrlimit(resource.RLIMIT_NOFILE, newlimit)
super(test_schema03, self).setUp()
def setUpConnectionOpen(self, dir):
conn = wiredtiger.wiredtiger_open(dir,
'create,cache_size=100m,session_max=1000')
self.pr(`conn`)
return conn
def tearDown(self):
super(test_schema03, self).tearDown()
resource.setrlimit(resource.RLIMIT_NOFILE, self.origFileLimit)
def gen_formats(self, rand, n, iskey):
result = ''
for i in range(0, n):
if rand.rand_range(0, 2) == 0:
result += 'S'
else:
result += 'i'
return result
def show_python(self, s):
if self.SHOW_PYTHON:
if self.SHOW_PYTHON_ONLY_TABLE == None or self.current_table in self.SHOW_PYTHON_ONLY_TABLE:
if self.SHOW_PYTHON_ONLY_SCEN == None or self.scenario_number in self.SHOW_PYTHON_ONLY_SCEN:
print ' ' + s
def join_names(self, sep, prefix, list):
return sep.join([prefix + str(val) for val in list])
def create(self, what, tablename, whatname, columnlist, extra_args=''):
createarg = what + ":" + tablename + ":" + whatname
colarg = self.join_names(',', 'c', columnlist)
self.show_python("self.session.create('" + createarg + "', 'columns=(" + colarg + ")" + extra_args + "')")
result = self.session.create(createarg,
"columns=(" + colarg + ")" + extra_args)
self.assertEqual(result, 0)
def finished_step(self, name):
if self.s_restart == name:
print " # Reopening connection at step: " + name
self.reopen_conn()
def test_schema(self):
rand = suite_random.suite_random()
if self.SHOW_PYTHON:
print ' ################################################'
print ' # Running scenario ' + str(self.scenario_number)
ntables = self.s_ntable
# Report known limitations in the test,
# we'll work around these later, in a loop where we don't want to print.
self.KNOWN_LIMITATION('Indices created after data population will have no entries')
self.KNOWN_LIMITATION('Column groups created after indices confuses things')
# Column groups are created in two different times.
# We call these two batches 'createsets'.
# So we don't have the exactly the same number of column groups
# for each table, for tests that indicate >1 colgroup, we
# increase the number of column groups for each table
tabconfigs = []
for i in range(0, ntables):
self.current_table = i
tc = tabconfig()
tc.tablename = 't' + str(i)
tc.tableidx = i
tabconfigs.append(tc)
for createset in range(0, 2):
ncg = self.s_colgroup[createset]
if ncg > 1:
ncg += i
for k in range(0, ncg):
thiscg = cgconfig()
thiscg.createset = createset
# KNOWN LIMITATION: Column groups created after
# indices confuses things. So for now, put all
# column group creation in the first set.
# Remove this statement when the limitation is fixed.
thiscg.createset = 0
# END KNOWN LIMITATION
thiscg.cgname = 'g' + str(len(tc.cglist))
tc.cglist.append(thiscg)
# The same idea for indices, except that we create them in
# three sets
for createset in range(0, 3):
nindex = self.s_index[createset]
if nindex > 1:
nindex += i
for k in range(0, nindex):
thisidx = idxconfig()
thisidx.createset = createset
thisidx.idxname = 'i' + str(len(tc.idxlist))
thisidx.tab = tc
tc.idxlist.append(thisidx)
# We'll base the number of key/value columns
# loosely on the number of column groups and indices.
colgroups = len(tc.cglist)
indices = len(tc.idxlist)
nall = colgroups * 2 + indices
k = rand.rand_range(1, nall)
v = rand.rand_range(0, nall)
# we need at least one value per column group
if v < colgroups:
v = colgroups
tc.nkeys = k
tc.nvalues = v
tc.keyformats = self.gen_formats(rand, tc.nkeys, True)
tc.valueformats = self.gen_formats(rand, tc.nvalues, False)
# Simple naming (we'll test odd naming elsewhere):
# tables named 't0' --> 't<N>'
# within each table:
# columns named 'c0' --> 'c<N>'
# colgroups named 'g0' --> 'g<N>'
# indices named 'i0' --> 'i<N>'
config = ""
config += "key_format=" + tc.keyformats
config += ",value_format=" + tc.valueformats
config += ",columns=("
for j in range(0, tc.nkeys + tc.nvalues):
if j != 0:
config += ","
config += "c" + str(j)
config += "),colgroups=("
for j in range(0, len(tc.cglist)):
if j != 0:
config += ","
config += "g" + str(j)
config += ")"
config += self.s_extra_table_args
# indices are not declared here
self.show_python("self.session.create('table:" + tc.tablename + "', '" + config + "')")
self.session.create("table:" + tc.tablename, config)
tc.columns_for_groups(range(tc.nkeys, tc.nkeys + tc.nvalues))
tc.columns_for_indices(range(0, tc.nkeys + tc.nvalues))
self.finished_step('table')
for createset in (0, 1):
# Create column groups in this set
# e.g. self.session.create("colgroup:t0:g1", "columns=(c3,c4)")
for tc in tabconfigs:
self.current_table = tc.tableidx
for cg in tc.cglist:
if cg.createset == createset:
self.create('colgroup', tc.tablename, cg.cgname, cg.columns)
self.finished_step('colgroup' + str(createset))
# Create indices in this set
# e.g. self.session.create("index:t0:i1", "columns=(c3,c4)")
for tc in tabconfigs:
self.current_table = tc.tableidx
for idx in tc.idxlist:
if idx.createset == createset:
self.create('index', tc.tablename, idx.idxname, idx.columns, self.s_index_args)
self.finished_step('index' + str(createset))
# populate first batch
for tc in tabconfigs:
self.current_table = tc.tableidx
max = rand.rand_range(0, self.nentries)
self.populate(tc, xrange(0, max))
self.finished_step('populate0')
#TODO
# Create indices in third set
# for tc in tabconfigs:
# for idx in tc.idxlist:
# if idx.createset == 2:
# self.create('index', tc.tablename, idx.idxname, idx.columns)
self.finished_step('index2')
# populate second batch
for tc in tabconfigs:
self.current_table = tc.tableidx
self.populate(tc, xrange(tc.nentries, self.nentries))
self.finished_step('populate1')
for tc in tabconfigs:
self.current_table = tc.tableidx
self.check_entries(tc)
def populate(self, tc, insertrange):
self.show_python("cursor = self.session.open_cursor('table:" + tc.tablename + "', None, None)")
cursor = self.session.open_cursor('table:' + tc.tablename, None, None)
for i in insertrange:
key = tc.gen_keys(i)
val = tc.gen_values(i)
self.show_python("cursor.set_key(*" + str(key) + ")")
cursor.set_key(*key)
self.show_python("cursor.set_value(*" + str(val) + ")")
cursor.set_value(*val)
self.show_python("cursor.insert()")
cursor.insert()
tc.nentries += 1
self.show_python("cursor.close()")
cursor.close()
def check_one(self, name, cursor, key, val):
keystr = str(key)
valstr = str(val)
self.show_python('# search[' + name + '](' + keystr + ')')
self.show_python("cursor.set_key(*" + keystr + ")")
cursor.set_key(*key)
self.show_python("ok = cursor.search()")
ok = cursor.search()
self.show_python("self.assertEqual(ok, 0)")
self.assertEqual(ok, 0)
self.show_python("self.assertEqual(" + keystr + ", cursor.get_keys())")
self.assertEqual(key, cursor.get_keys())
self.show_python("self.assertEqual(" + valstr + ", cursor.get_values())")
self.assertEqual(val, cursor.get_values())
def check_entries(self, tc):
"""
Verify entries in the primary and index table
related to the tabconfig.
"""
self.show_python('# check_entries: ' + tc.tablename)
self.show_python("cursor = self.session.open_cursor('table:" + tc.tablename + "', None, None)")
cursor = self.session.open_cursor('table:' + tc.tablename, None, None)
count = 0
for x in cursor:
count += 1
self.assertEqual(count, tc.nentries)
for i in range(0, tc.nentries):
key = tc.gen_keys(i)
val = tc.gen_values(i)
self.check_one(tc.tablename, cursor, key, val)
cursor.close()
self.show_python("cursor.close()")
# for each index, check each entry
for idx in tc.idxlist:
# KNOWN LIMITATION: Indices created after data population
# will have no entries, so don't bother with them here
# Remove these statements when the limitation is fixed.
if idx.createset == 2:
continue
# END KNOWN LIMITATION
# Although it's possible to open an index on some partial
# list of columns, we'll keep it simple here, and always
# use all columns.
full_idxname = 'index:' + tc.tablename + ':' + idx.idxname
cols = '(' + ','.join([('c' + str(x)) for x in range(tc.nkeys, tc.nvalues + tc.nkeys)]) + ')'
self.show_python('# check_entries: ' + full_idxname + cols)
self.show_python("cursor = self.session.open_cursor('" + full_idxname + cols + "', None, None)")
cursor = self.session.open_cursor(full_idxname + cols, None, None)
count = 0
for x in cursor:
count += 1
self.assertEqual(count, tc.nentries)
for i in range(0, tc.nentries):
key = idx.gen_keys(i)
val = tc.gen_values(i)
self.check_one(full_idxname, cursor, key, val)
cursor.close()
self.show_python("cursor.close()")
class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess):
tablebase = 'test_sweep01'
uri = 'table:' + tablebase
numfiles = 500
numkv = 100
ckpt_list = [
('off', dict(ckpt=0)),
('on', dict(ckpt=20)),
]
types = [
('row',
dict(tabletype='row', create_params='key_format=i,value_format=i')),
('var',
dict(tabletype='var', create_params='key_format=r,value_format=i')),
('fix',
dict(tabletype='fix', create_params='key_format=r,value_format=8t')),
]
scenarios = number_scenarios(
prune_scenarios(multiply_scenarios('.', types, ckpt_list), 1, 100))
# Overrides WiredTigerTestCase
def setUpConnectionOpen(self, dir):
self.home = dir
self.backup_dir = os.path.join(self.home, "WT_BACKUP")
conn_params = \
',create,error_prefix="%s: ",' % self.shortid() + \
'checkpoint=(wait=%d),' % self.ckpt + \
'statistics=(fast),'
# print "Creating conn at '%s' with config '%s'" % (dir, conn_params)
try:
conn = wiredtiger_open(dir, conn_params)
except wiredtiger.WiredTigerError as e:
print "Failed conn at '%s' with config '%s'" % (dir, conn_params)
self.pr( ` conn `)
self.session2 = conn.open_session()
return conn
def test_ops(self):
#
# Set up numfiles with numkv entries. We just want some data in there
# we don't care what it is.
#
for f in range(self.numfiles):
uri = '%s.%d' % (self.uri, f)
# print "Creating %s with config '%s'" % (uri, self.create_params)
self.session.create(uri, self.create_params)
c = self.session.open_cursor(uri, None)
c.set_value(1)
for k in range(self.numkv):
c.set_key(k + 1)
c.insert()
c.close()
if f % 20 == 0:
time.sleep(1)
stat_cursor = self.session.open_cursor('statistics:', None, None)
close1 = stat_cursor[stat.conn.dh_conn_handles][2]
sweep1 = stat_cursor[stat.conn.dh_conn_sweeps][2]
sclose1 = stat_cursor[stat.conn.dh_session_handles][2]
ssweep1 = stat_cursor[stat.conn.dh_session_sweeps][2]
tod1 = stat_cursor[stat.conn.dh_conn_tod][2]
ref1 = stat_cursor[stat.conn.dh_conn_ref][2]
nfile1 = stat_cursor[stat.conn.file_open][2]
stat_cursor.close()
# Inactive time on a handle must be a minute or more.
# At some point the sweep thread will run and set the time of death
# to be a minute later. So sleep 2 minutes to make sure it has run
# enough times to timeout the handles.
uri = '%s.test' % self.uri
self.session.create(uri, self.create_params)
#
# Keep inserting data to keep at least one handle active and give
# checkpoint something to do. Make sure checkpoint doesn't adjust
# the time of death for inactive handles.
#
c = self.session.open_cursor(uri, None)
k = 0
sleep = 0
while sleep < 120:
k = k + 1
c.set_key(k)
c.set_value(1)
c.insert()
sleep += 10
time.sleep(10)
c.close()
stat_cursor = self.session.open_cursor('statistics:', None, None)
close2 = stat_cursor[stat.conn.dh_conn_handles][2]
sweep2 = stat_cursor[stat.conn.dh_conn_sweeps][2]
sclose2 = stat_cursor[stat.conn.dh_session_handles][2]
ssweep2 = stat_cursor[stat.conn.dh_session_sweeps][2]
nfile2 = stat_cursor[stat.conn.file_open][2]
tod2 = stat_cursor[stat.conn.dh_conn_tod][2]
ref2 = stat_cursor[stat.conn.dh_conn_ref][2]
stat_cursor.close()
# print "checkpoint: " + str(self.ckpt)
# print "nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)
# print "close1: " + str(close1) + " close2: " + str(close2)
# print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
# print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
# print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
# print "tod1: " + str(tod1) + " tod2: " + str(tod2)
# print "ref1: " + str(ref1) + " ref2: " + str(ref2)
#
# The files are all closed. Check that sweep did its work even
# in the presence of recent checkpoints.
#
if (close1 >= close2):
print "XX: close1: " + str(close1) + " close2: " + str(close2)
print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
print "tod1: " + str(tod1) + " tod2: " + str(tod2)
print "ref1: " + str(ref1) + " ref2: " + str(ref2)
print "nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)
self.assertEqual(close1 < close2, True)
if (sweep1 >= sweep2):
print "close1: " + str(close1) + " close2: " + str(close2)
print "XX: sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
print "tod1: " + str(tod1) + " tod2: " + str(tod2)
print "ref1: " + str(ref1) + " ref2: " + str(ref2)
self.assertEqual(sweep1 < sweep2, True)
if (nfile2 >= nfile1):
print "close1: " + str(close1) + " close2: " + str(close2)
print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
print "tod1: " + str(tod1) + " tod2: " + str(tod2)
print "ref1: " + str(ref1) + " ref2: " + str(ref2)
print "XX: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)
self.assertEqual(nfile2 < nfile1, True)
# The only files that should be left is the metadata and the active one.
if (nfile2 != 2):
print "close1: " + str(close1) + " close2: " + str(close2)
print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)
print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)
print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)
print "tod1: " + str(tod1) + " tod2: " + str(tod2)
print "ref1: " + str(ref1) + " ref2: " + str(ref2)
print "XX2: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)
self.assertEqual(nfile2 == 2, True)
