class test_tiered01(wttest.WiredTigerTestCase):
K = 1024
M = 1024 * K
G = 1024 * M
uri = "table:test_tiered01"
chunk_size_scenarios = wtscenario.quick_scenarios('s_chunk_size',
[1 * M, 20 * M, None],
[0.6, 0.6, 0.6])
# Occasionally add a lot of records, so that merges (and bloom) happen.
record_count_scenarios = wtscenario.quick_scenarios(
'nrecs', [10, 10000], [0.9, 0.1])
config_vars = [
'chunk_size',
]
scenarios = wtscenario.make_scenarios(chunk_size_scenarios,
record_count_scenarios,
prune=100,
prunelong=500)
# Test create of an object.
def test_tiered(self):
self.session.create('file:first.wt', 'key_format=S')
self.session.create('file:second.wt', 'key_format=S')
args = 'type=tiered,key_format=S'
args += ',tiered=(' # Start the tiered configuration options.
args += 'tiers=("file:first.wt", "file:second.wt"),'
# add names to args, e.g. args += ',session_max=30'
for var in self.config_vars:
value = getattr(self, 's_' + var)
if value != None:
if var == 'verbose':
value = '[' + str(value) + ']'
value = {True: 'true', False: 'false'}.get(value, value)
args += ',' + var + '=' + str(value)
args += ')' # Close the tiered configuration option group
self.verbose(
3,
'Test tiered with config: ' + args + ' count: ' + str(self.nrecs))
SimpleDataSet(self, self.uri, self.nrecs, config=args).populate()
# self.session.drop(self.uri)
# It is an error to configure a tiered table with no tiers
def test_no_tiers(self):
msg = '/tiered table must specify at least one tier/'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: self.session.create(
self.uri, 'type=tiered,key_format=S,tiered=(tiers=())'), msg)
class test_tiered02(wttest.WiredTigerTestCase):
K = 1024
M = 1024 * K
G = 1024 * M
uri = "file:test_tiered02"
# Occasionally add a lot of records, so that merges (and bloom) happen.
record_count_scenarios = wtscenario.quick_scenarios(
'nrecs', [10, 10000], [0.9, 0.1])
scenarios = wtscenario.make_scenarios(record_count_scenarios,
prune=100,
prunelong=500)
# Test drop of an object.
def test_tiered(self):
args = 'key_format=S,block_allocation=log-structured'
self.verbose(
3, 'Test log-structured allocation with config: ' + args +
' count: ' + str(self.nrecs))
#ds = SimpleDataSet(self, self.uri, self.nrecs, config=args)
ds = SimpleDataSet(self, self.uri, 10, config=args)
ds.populate()
self.session.checkpoint()
ds = SimpleDataSet(self, self.uri, 10000, config=args)
ds.populate()
self.reopen_conn()
ds = SimpleDataSet(self, self.uri, 1000, config=args)
ds.populate()
class test_tiered03(wttest.WiredTigerTestCase):
K = 1024
M = 1024 * K
G = 1024 * M
uri = 'file:test_tiered03'
# Occasionally add a lot of records, so that merges (and bloom) happen.
record_count_scenarios = wtscenario.quick_scenarios(
'nrecs', [10, 10000], [0.9, 0.1])
scenarios = wtscenario.make_scenarios(record_count_scenarios, prune=100, prunelong=500)
# Test sharing data between a primary and a secondary
def test_sharing(self):
args = 'block_allocation=log-structured'
self.verbose(3,
'Test log-structured allocation with config: ' + args + ' count: ' + str(self.nrecs))
ds = SimpleDataSet(self, self.uri, 10, config=args)
ds.populate()
ds.check()
self.session.checkpoint()
ds.check()
# Create a secondary database
dir2 = os.path.join(self.home, 'SECONDARY')
os.mkdir(dir2)
conn2 = self.setUpConnectionOpen(dir2)
session2 = conn2.open_session()
# Reference the tree from the secondary:
metac = self.session.open_cursor('metadata:')
metac2 = session2.open_cursor('metadata:', None, 'readonly=0')
uri2 = self.uri[:5] + '../' + self.uri[5:]
metac2[uri2] = metac[self.uri] + ",readonly=1"
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
newds = SimpleDataSet(self, self.uri, 10000, config=args)
newds.populate()
newds.check()
self.session.checkpoint()
newds.check()
# Check we can still read from the last checkpoint
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
# Bump to new checkpoint
origmeta = metac[self.uri]
checkpoint = re.search(r',checkpoint=\(.+?\)\)', origmeta).group(0)[1:]
self.pr('Orig checkpoint: ' + checkpoint)
session2.alter(uri2, checkpoint)
self.pr('New metadata on secondaery: ' + metac2[uri2])
# Check that we can see the new data
cursor2 = session2.open_cursor(uri2)
newds.check_cursor(cursor2)
class test_lsm01(wttest.WiredTigerTestCase):
K = 1024
M = 1024 * K
G = 1024 * M
uri = "lsm:test_lsm01"
chunk_size_scenarios = wtscenario.quick_scenarios('s_chunk_size',
[1 * M, 20 * M, None],
[0.6, 0.6, 0.6])
merge_max_scenarios = wtscenario.quick_scenarios('s_merge_max',
[2, 10, 20, None], None)
bloom_scenarios = wtscenario.quick_scenarios('s_bloom',
[True, False, None], None)
bloom_bit_scenarios = wtscenario.quick_scenarios('s_bloom_bit_count',
[2, 8, 20, None], None)
bloom_hash_scenarios = wtscenario.quick_scenarios('s_bloom_hash_count',
[2, 10, 20, None], None)
# Occasionally add a lot of records, so that merges (and bloom) happen.
record_count_scenarios = wtscenario.quick_scenarios(
'nrecs', [10, 10000], [0.9, 0.1])
config_vars = [
'chunk_size', 'merge_max', 'bloom', 'bloom_bit_count',
'bloom_hash_count'
]
scenarios = wtscenario.make_scenarios(chunk_size_scenarios,
merge_max_scenarios,
bloom_scenarios,
bloom_bit_scenarios,
bloom_hash_scenarios,
record_count_scenarios,
prune=100,
prunelong=500)
# Test drop of an object.
def test_lsm(self):
args = 'key_format=S'
args += ',lsm=(' # Start the LSM configuration options.
# add names to args, e.g. args += ',session_max=30'
for var in self.config_vars:
value = getattr(self, 's_' + var)
if value != None:
if var == 'verbose':
value = '[' + str(value) + ']'
if value == True:
value = 'true'
if value == False:
value = 'false'
args += ',' + var + '=' + str(value)
args += ')' # Close the LSM configuration option group
self.verbose(
3, 'Test LSM with config: ' + args + ' count: ' + str(self.nrecs))
SimpleDataSet(self, self.uri, self.nrecs).populate()
class test_config03(test_base03.test_base03):
K = 1024
M = 1024 * K
G = 1024 * M
cache_size_scenarios = wtscenario.quick_scenarios(
's_cache_size', [1 * M, 20 * M, 100 * M, 1 * G, None],
[0.6, 0.6, 0.6, 0.6, 0.6])
create_scenarios = wtscenario.quick_scenarios('s_create',
[True, False, None],
[1.0, 0.2, 0.3])
error_prefix_scenarios = wtscenario.quick_scenarios(
's_error_prefix', [None, "errpfx:"], [1.0, 0.2])
# eviction_target < eviction_trigger -- checked later
eviction_target_scenarios = wtscenario.quick_scenarios(
's_eviction_target', [10, 40, 85, 98], None)
eviction_trigger_scenarios = wtscenario.quick_scenarios(
's_eviction_trigger', [50, 90, 95, 99], None)
hazard_max_scenarios = wtscenario.quick_scenarios('s_hazard_max',
[15, 50, 500],
[0.4, 0.8, 0.8])
multiprocess_scenarios = wtscenario.quick_scenarios(
's_multiprocess', [True, False], [1.0, 1.0])
session_max_scenarios = wtscenario.quick_scenarios('s_session_max',
[3, 30, 300], None)
transactional_scenarios = wtscenario.quick_scenarios(
's_transactional', [True, False], [0.2, 1.0])
# Note: we are not using any truly verbose scenarios until we have
# a way to redirect verbose output to a file in Python.
#
#verbose_scenarios = wtscenario.quick_scenarios('s_verbose',
# ['block', 'evict,evictserver', 'fileops,hazard,mutex',
# 'read,readserver,reconcile,salvage','verify,write',''], None)
verbose_scenarios = wtscenario.quick_scenarios('s_verbose', [None], None)
config_vars = [
'cache_size', 'create', 'error_prefix', 'eviction_target',
'eviction_trigger', 'hazard_max', 'multiprocess', 'session_max',
'verbose'
]
all_scenarios = wtscenario.multiply_scenarios(
'_', cache_size_scenarios, create_scenarios, error_prefix_scenarios,
eviction_target_scenarios, eviction_trigger_scenarios,
hazard_max_scenarios, multiprocess_scenarios, session_max_scenarios,
transactional_scenarios, verbose_scenarios)
scenarios = wtscenario.prune_scenarios(all_scenarios, 1000)
scenarios = wtscenario.number_scenarios(scenarios)
#wttest.WiredTigerTestCase.printVerbose(2, 'test_config03: running ' + \
# str(len(scenarios)) + ' of ' + \
# str(len(all_scenarios)) + ' possible scenarios')
def setUpConnectionOpen(self, dir):
args = ''
# add names to args, e.g. args += ',session_max=30'
for var in self.config_vars:
value = getattr(self, 's_' + var)
if value != None:
if var == 'verbose':
value = '[' + str(value) + ']'
if value == True:
value = 'true'
if value == False:
value = 'false'
args += ',' + var + '=' + str(value)
args += ','
self.pr('wiredtiger_open with args: ' + args)
expect_fail = False
successargs = args
if self.s_create == False:
successargs = successargs.replace(',create=false,', ',create,')
expect_fail = True
fail_msg = '/(No such file or directory|The system cannot find the file specified)/'
elif self.s_create == None:
successargs = successargs + 'create=true,'
expect_fail = True
fail_msg = '/(No such file or directory|The system cannot find the file specified)/'
if self.s_eviction_target >= self.s_eviction_trigger:
# construct args that guarantee that target < trigger
# we know that trigger >= 1
repfrom = ',eviction_target=' + str(self.s_eviction_target)
repto = ',eviction_target=' + str(self.s_eviction_trigger - 1)
successargs = successargs.replace(repfrom, repto)
if not expect_fail:
expect_fail = True
fail_msg = \
'/eviction target must be lower than the eviction trigger/'
if expect_fail:
self.verbose(3, 'wiredtiger_open (should fail) with args: ' + args)
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: wiredtiger.wiredtiger_open(dir, args), fail_msg)
args = successargs
self.verbose(3, 'wiredtiger_open with args: ' + args)
conn = wiredtiger.wiredtiger_open(dir, args)
self.pr( ` conn `)
return conn
class test_tiered03(wttest.WiredTigerTestCase, TieredConfigMixin):
K = 1024
M = 1024 * K
G = 1024 * M
# TODO: tiered: change this to a table: URI, otherwise we are
# not using tiered files. The use of a second directory for
# sharing would probably need to be reworked.
uri = 'file:test_tiered03'
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered03',
tiered_only=True)
# Occasionally add a lot of records to vary the amount of work flush does.
record_count_scenarios = wtscenario.quick_scenarios(
'nrecs', [10, 10000], [0.9, 0.1])
scenarios = wtscenario.make_scenarios(storage_sources, record_count_scenarios,\
prune=100, prunelong=500)
absolute_bucket_dir = None # initialied in conn_config to an absolute path
def conn_config(self):
bucket_ret = self.bucket
# The bucket format for the S3 store is the name and the region separated by a semi-colon.
if self.ss_name == 's3_store':
cache_dir = self.bucket[:self.bucket.find(';')] + '-cache'
else:
cache_dir = self.bucket + '-cache'
# We have multiple connections that want to share a bucket.
# For the directory store, the first time this function is called, we'll
# establish the absolute path for the bucket, and always use that for
# the bucket name.
# The cache directory name is a relative one, so it won't be shared
# between connections.
if self.ss_name == 'dir_store':
if self.absolute_bucket_dir == None:
self.absolute_bucket_dir = os.path.join(
os.getcwd(), self.bucket)
os.mkdir(self.absolute_bucket_dir)
bucket_ret = self.absolute_bucket_dir
return get_conn_config(self) + 'cache_directory=%s)' % cache_dir
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
# Test sharing data between a primary and a secondary
def test_sharing(self):
# FIXME: WT-8235 Enable the test once file containing transaction ids is supported.
self.skipTest(
'Sharing the checkpoint file containing transaction ids is not supported'
)
ds = SimpleDataSet(self, self.uri, 10)
ds.populate()
ds.check()
self.session.checkpoint()
ds.check()
# Create a secondary database
dir2 = os.path.join(self.home, 'SECONDARY')
os.mkdir(dir2)
conn2 = self.setUpConnectionOpen(dir2)
session2 = conn2.open_session()
# Reference the tree from the secondary:
metac = self.session.open_cursor('metadata:')
metac2 = session2.open_cursor('metadata:', None, 'readonly=0')
uri2 = self.uri[:5] + '../' + self.uri[5:]
metac2[uri2] = metac[self.uri] + ",readonly=1"
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
newds = SimpleDataSet(self, self.uri, 10000)
newds.populate()
newds.check()
self.session.checkpoint()
newds.check()
# Check we can still read from the last checkpoint
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
# Bump to new checkpoint
origmeta = metac[self.uri]
checkpoint = re.search(r',checkpoint=\(.+?\)\)', origmeta).group(0)[1:]
self.pr('Orig checkpoint: ' + checkpoint)
session2.alter(uri2, checkpoint)
self.pr('New metadata on secondaery: ' + metac2[uri2])
# Check that we can see the new data
cursor2 = session2.open_cursor(uri2)
newds.check_cursor(cursor2)
class test_schema03(wttest.WiredTigerTestCase):
"""
Test schemas - a 'predictably random' assortment of columns,
column groups and indices are created within tables, and are
created in various orders as much as the API allows. On some runs
the connection will be closed and reopened at a particular point
to test that the schemas (and data) are saved and read correctly.
The test is run multiple times, using scenarios.
The test always follows these steps:
- table: create tables
- colgroup0: create (some) colgroups
- index0: create (some) indices
- colgroup1: create (more) colgroups
- index1: create (more) indices
- populate0: populate 1st time
- index2: create (more) indices
- populate1: populate 2nd time (more key/values)
- check: check key/values
The variations represented by scenarios are:
- how many tables to create
- how many colgroups to create at each step (may be 0)
- how many indices to create at each step (may be 0)
- between each step, whether to close/reopen the connection
"""
# Boost cache size and number of sessions for this test
conn_config = 'cache_size=100m,session_max=1000'
################################################################
# These three variables can be altered to help generate
# and pare down failing test cases.
# Set to true to get python test program fragment on stdout,
# used by show_python() below.
SHOW_PYTHON = False
# When SHOW_PYTHON is set, we print an enormous amount of output.
# To only print for a given scenario, set this
SHOW_PYTHON_ONLY_SCEN = None # could be e.g. [2] or [0,1]
# To print verbosely for only a given table, set this
SHOW_PYTHON_ONLY_TABLE = None # could be e.g. [2] or [0,1]
################################################################
# Set whenever we are working with a table
current_table = None
nentries = 50
# We need to have a large number of open files available
# to run this test. We probably don't need quite this many,
# but boost it up to this limit anyway.
OPEN_FILE_LIMIT = 1000
restart_scenarios = [
('table', dict(s_restart=['table'], P=0.3)),
('colgroup0', dict(s_restart=['colgroup0'], P=0.3)),
('index0', dict(s_restart=['index0'], P=0.3)),
('colgroup1', dict(s_restart=['colgroup1'], P=0.3)),
('index1', dict(s_restart=['index1'], P=0.3)),
('populate0', dict(s_restart=['populate0'], P=0.3)),
('index2', dict(s_restart=['index2'], P=0.3)),
('populate1', dict(s_restart=['populate1'], P=0.3)),
('ipop', dict(s_restart=['index0', 'populate0'], P=0.3)),
('all',
dict(s_restart=[
'table', 'colgroup0', 'index0', 'colgroup1', 'index1',
'populate0', 'index2', 'populate1'
],
P=1.0)),
]
ntable_scenarios = wtscenario.quick_scenarios('s_ntable', [1, 2, 5, 8],
[1.0, 0.4, 0.5, 0.5])
ncolgroup_scenarios = wtscenario.quick_scenarios(
's_colgroup', [[1, 0], [0, 1], [2, 4], [8, 5]], [1.0, 0.2, 0.3, 1.0])
nindex_scenarios = wtscenario.quick_scenarios(
's_index', [[1, 1, 1], [3, 2, 1], [5, 1, 3]], [1.0, 0.5, 1.0])
idx_args_scenarios = wtscenario.quick_scenarios(
's_index_args', ['', ',type=file', ',type=lsm'], [0.5, 0.3, 0.2])
table_args_scenarios = wtscenario.quick_scenarios(
's_extra_table_args', ['', ',type=file', ',type=lsm'], [0.5, 0.3, 0.2])
scenarios = wtscenario.make_scenarios(restart_scenarios,
ntable_scenarios,
ncolgroup_scenarios,
nindex_scenarios,
idx_args_scenarios,
table_args_scenarios,
prune=30)
# Note: the set can be reduced here for debugging, e.g.
# scenarios = scenarios[40:44]
# or
# scenarios = [ scenarios[0], scenarios[30], scenarios[40] ]
#wttest.WiredTigerTestCase.printVerbose(2, 'test_schema03: running ' + \
# str(len(scenarios)) + ' of ' + \
# str(len(all_scenarios)) + ' possible scenarios')
# This test requires a large number of open files.
# Increase our resource limits before we start
def setUp(self):
if os.name == "nt":
self.skipTest('Unix specific test skipped on Windows')
self.origFileLimit = resource.getrlimit(resource.RLIMIT_NOFILE)
newlimit = (self.OPEN_FILE_LIMIT, self.origFileLimit[1])
if newlimit[0] > newlimit[1]:
self.skipTest('Require %d open files, only %d available' %
newlimit)
resource.setrlimit(resource.RLIMIT_NOFILE, newlimit)
super(test_schema03, self).setUp()
def tearDown(self):
super(test_schema03, self).tearDown()
resource.setrlimit(resource.RLIMIT_NOFILE, self.origFileLimit)
def gen_formats(self, rand, n, iskey):
result = ''
for i in range(0, n):
if rand.rand_range(0, 2) == 0:
result += 'S'
else:
result += 'i'
return result
def show_python(self, s):
if self.SHOW_PYTHON:
if self.SHOW_PYTHON_ONLY_TABLE == None or self.current_table in self.SHOW_PYTHON_ONLY_TABLE:
if self.SHOW_PYTHON_ONLY_SCEN == None or self.scenario_number in self.SHOW_PYTHON_ONLY_SCEN:
print(' ' + s)
def join_names(self, sep, prefix, list):
return sep.join([prefix + str(val) for val in list])
def create(self, what, tablename, whatname, columnlist, extra_args=''):
createarg = what + ":" + tablename + ":" + whatname
colarg = self.join_names(',', 'c', columnlist)
self.show_python("self.session.create('" + createarg +
"', 'columns=(" + colarg + ")" + extra_args + "')")
result = self.session.create(createarg,
"columns=(" + colarg + ")" + extra_args)
self.assertEqual(result, 0)
def finished_step(self, name):
if self.s_restart == name:
print(" # Reopening connection at step: " + name)
self.reopen_conn()
def test_schema(self):
rand = suite_random.suite_random()
if self.SHOW_PYTHON:
print(' ################################################')
print(' # Running scenario ' + str(self.scenario_number))
ntables = self.s_ntable
# Report known limitations in the test,
# we'll work around these later, in a loop where we don't want to print.
self.KNOWN_LIMITATION(
'Column groups created after indices confuses things')
# Column groups are created in two different times.
# We call these two batches 'createsets'.
# So we don't have the exactly the same number of column groups
# for each table, for tests that indicate >1 colgroup, we
# increase the number of column groups for each table
tabconfigs = []
for i in range(0, ntables):
self.current_table = i
tc = tabconfig()
tc.tablename = 't' + str(i)
tc.tableidx = i
tabconfigs.append(tc)
for createset in range(0, 2):
ncg = self.s_colgroup[createset]
if ncg > 1:
ncg += i
for k in range(0, ncg):
thiscg = cgconfig()
thiscg.createset = createset
# KNOWN LIMITATION: Column groups created after
# indices confuses things. So for now, put all
# column group creation in the first set.
# Remove this statement when the limitation is fixed.
thiscg.createset = 0
# END KNOWN LIMITATION
thiscg.cgname = 'g' + str(len(tc.cglist))
tc.cglist.append(thiscg)
# The same idea for indices, except that we create them in
# three sets
for createset in range(0, 3):
nindex = self.s_index[createset]
if nindex > 1:
nindex += i
for k in range(0, nindex):
thisidx = idxconfig()
thisidx.createset = createset
thisidx.idxname = 'i' + str(len(tc.idxlist))
thisidx.tab = tc
tc.idxlist.append(thisidx)
# We'll base the number of key/value columns
# loosely on the number of column groups and indices.
colgroups = len(tc.cglist)
indices = len(tc.idxlist)
nall = colgroups * 2 + indices
k = rand.rand_range(1, nall)
v = rand.rand_range(0, nall)
# we need at least one value per column group
if v < colgroups:
v = colgroups
tc.nkeys = k
tc.nvalues = v
tc.keyformats = self.gen_formats(rand, tc.nkeys, True)
tc.valueformats = self.gen_formats(rand, tc.nvalues, False)
# Simple naming (we'll test odd naming elsewhere):
# tables named 't0' --> 't<N>'
# within each table:
# columns named 'c0' --> 'c<N>'
# colgroups named 'g0' --> 'g<N>'
# indices named 'i0' --> 'i<N>'
config = ""
config += "key_format=" + tc.keyformats
config += ",value_format=" + tc.valueformats
config += ",columns=("
for j in range(0, tc.nkeys + tc.nvalues):
if j != 0:
config += ","
config += "c" + str(j)
config += "),colgroups=("
for j in range(0, len(tc.cglist)):
if j != 0:
config += ","
config += "g" + str(j)
config += ")"
config += self.s_extra_table_args
# indices are not declared here
self.show_python("self.session.create('table:" + tc.tablename +
"', '" + config + "')")
self.session.create("table:" + tc.tablename, config)
tc.columns_for_groups(list(range(tc.nkeys, tc.nkeys + tc.nvalues)))
tc.columns_for_indices(list(range(0, tc.nkeys + tc.nvalues)))
self.finished_step('table')
for createset in (0, 1):
# Create column groups in this set
# e.g. self.session.create("colgroup:t0:g1", "columns=(c3,c4)")
for tc in tabconfigs:
self.current_table = tc.tableidx
for cg in tc.cglist:
if cg.createset == createset:
self.create('colgroup', tc.tablename, cg.cgname,
cg.columns)
self.finished_step('colgroup' + str(createset))
# Create indices in this set
# e.g. self.session.create("index:t0:i1", "columns=(c3,c4)")
for tc in tabconfigs:
self.current_table = tc.tableidx
for idx in tc.idxlist:
if idx.createset == createset:
self.create('index', tc.tablename, idx.idxname,
idx.columns, self.s_index_args)
self.finished_step('index' + str(createset))
# populate first batch
for tc in tabconfigs:
self.current_table = tc.tableidx
max = rand.rand_range(0, self.nentries)
self.populate(tc, list(range(0, max)))
self.finished_step('populate0')
# Create indices in third set
for tc in tabconfigs:
for idx in tc.idxlist:
if idx.createset == 2:
self.create('index', tc.tablename, idx.idxname,
idx.columns)
self.finished_step('index2')
# populate second batch
for tc in tabconfigs:
self.current_table = tc.tableidx
self.populate(tc, list(range(tc.nentries, self.nentries)))
self.finished_step('populate1')
for tc in tabconfigs:
self.current_table = tc.tableidx
self.check_entries(tc)
def populate(self, tc, insertrange):
self.show_python("cursor = self.session.open_cursor('table:" +
tc.tablename + "', None, None)")
cursor = self.session.open_cursor('table:' + tc.tablename, None, None)
for i in insertrange:
key = tc.gen_keys(i)
val = tc.gen_values(i)
self.show_python("cursor.set_key(*" + str(key) + ")")
cursor.set_key(*key)
self.show_python("cursor.set_value(*" + str(val) + ")")
cursor.set_value(*val)
self.show_python("cursor.insert()")
cursor.insert()
tc.nentries += 1
self.show_python("cursor.close()")
cursor.close()
def check_one(self, name, cursor, key, val):
keystr = str(key)
valstr = str(val)
self.show_python('# search[' + name + '](' + keystr + ')')
self.show_python("cursor.set_key(*" + keystr + ")")
cursor.set_key(*key)
self.show_python("ok = cursor.search()")
ok = cursor.search()
self.show_python("self.assertEqual(ok, 0)")
self.assertEqual(ok, 0)
self.show_python("self.assertEqual(" + keystr + ", cursor.get_keys())")
self.assertEqual(key, cursor.get_keys())
self.show_python("self.assertEqual(" + valstr +
", cursor.get_values())")
self.assertEqual(val, cursor.get_values())
def check_entries(self, tc):
"""
Verify entries in the primary and index table
related to the tabconfig.
"""
self.show_python('# check_entries: ' + tc.tablename)
self.show_python("cursor = self.session.open_cursor('table:" +
tc.tablename + "', None, None)")
cursor = self.session.open_cursor('table:' + tc.tablename, None, None)
count = 0
for x in cursor:
count += 1
self.assertEqual(count, tc.nentries)
for i in range(0, tc.nentries):
key = tc.gen_keys(i)
val = tc.gen_values(i)
self.check_one(tc.tablename, cursor, key, val)
cursor.close()
self.show_python("cursor.close()")
# for each index, check each entry
for idx in tc.idxlist:
# Although it's possible to open an index on some partial
# list of columns, we'll keep it simple here, and always
# use all columns.
full_idxname = 'index:' + tc.tablename + ':' + idx.idxname
cols = '(' + ','.join([
('c' + str(x)) for x in range(tc.nkeys, tc.nvalues + tc.nkeys)
]) + ')'
self.show_python('# check_entries: ' + full_idxname + cols)
self.show_python("cursor = self.session.open_cursor('" +
full_idxname + cols + "', None, None)")
cursor = self.session.open_cursor(full_idxname + cols, None, None)
count = 0
for x in cursor:
count += 1
self.assertEqual(count, tc.nentries)
for i in range(0, tc.nentries):
key = idx.gen_keys(i)
val = tc.gen_values(i)
self.check_one(full_idxname, cursor, key, val)
cursor.close()
self.show_python("cursor.close()")
class test_tiered03(wttest.WiredTigerTestCase):
K = 1024
M = 1024 * K
G = 1024 * M
# TODO: tiered: change this to a table: URI, otherwise we are
# not using tiered files. The use of a second directory for
# sharing would probably need to be reworked.
uri = 'file:test_tiered03'
# Occasionally add a lot of records, so that merges (and bloom) happen.
record_count_scenarios = wtscenario.quick_scenarios(
'nrecs', [10, 10000], [0.9, 0.1])
scenarios = wtscenario.make_scenarios(record_count_scenarios,
prune=100,
prunelong=500)
auth_token = "test_token"
bucket = "mybucket"
bucket_prefix = "pfx_"
extension_name = "local_store"
prefix = "pfx-"
def conn_config(self):
if not os.path.exists(self.bucket):
os.mkdir(self.bucket)
return \
'statistics=(all),' + \
'tiered_storage=(auth_token=%s,' % self.auth_token + \
'bucket=%s,' % self.bucket + \
'bucket_prefix=%s,' % self.prefix + \
'name=%s)' % self.extension_name
# Load the local store extension, but skip the test if it is missing.
def conn_extensions(self, extlist):
extlist.skip_if_missing = True
extlist.extension('storage_sources', self.extension_name)
# Test sharing data between a primary and a secondary
def test_sharing(self):
ds = SimpleDataSet(self, self.uri, 10)
ds.populate()
ds.check()
self.session.checkpoint()
ds.check()
# Create a secondary database
dir2 = os.path.join(self.home, 'SECONDARY')
os.mkdir(dir2)
conn2 = self.setUpConnectionOpen(dir2)
session2 = conn2.open_session()
# Reference the tree from the secondary:
metac = self.session.open_cursor('metadata:')
metac2 = session2.open_cursor('metadata:', None, 'readonly=0')
uri2 = self.uri[:5] + '../' + self.uri[5:]
metac2[uri2] = metac[self.uri] + ",readonly=1"
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
newds = SimpleDataSet(self, self.uri, 10000)
newds.populate()
newds.check()
self.session.checkpoint()
newds.check()
# Check we can still read from the last checkpoint
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
# Bump to new checkpoint
origmeta = metac[self.uri]
checkpoint = re.search(r',checkpoint=\(.+?\)\)', origmeta).group(0)[1:]
self.pr('Orig checkpoint: ' + checkpoint)
session2.alter(uri2, checkpoint)
self.pr('New metadata on secondaery: ' + metac2[uri2])
# Check that we can see the new data
cursor2 = session2.open_cursor(uri2)
newds.check_cursor(cursor2)
class test_tiered03(wttest.WiredTigerTestCase):
K = 1024
M = 1024 * K
G = 1024 * M
# TODO: tiered: change this to a table: URI, otherwise we are
# not using tiered files. The use of a second directory for
# sharing would probably need to be reworked.
uri = 'file:test_tiered03'
# Occasionally add a lot of records, so that merges (and bloom) happen.
record_count_scenarios = wtscenario.quick_scenarios(
'nrecs', [10, 10000], [0.9, 0.1])
scenarios = wtscenario.make_scenarios(record_count_scenarios,
prune=100,
prunelong=500)
auth_token = "test_token"
bucket = "mybucket"
absolute_bucket_dir = None # initialied in conn_config to an absolute path
cache_dir = "mybucket-cache" # a relative pathname, it will not be shared
bucket_prefix = "pfx_"
extension_name = "local_store"
def conn_config(self):
# We have multiple connections that want to share a bucket directory.
# The first time this function is called, we'll establish the absolute
# path for the bucket, and always use that for the bucket name.
# The cache directory name is a relative one, so it won't be shared
# between connections.
if self.absolute_bucket_dir == None:
self.absolute_bucket_dir = os.path.join(os.getcwd(), self.bucket)
os.mkdir(self.absolute_bucket_dir)
return \
'tiered_storage=(auth_token=%s,' % self.auth_token + \
'bucket=%s,' % self.absolute_bucket_dir + \
'cache_directory=%s,' % self.cache_dir + \
'bucket_prefix=%s,' % self.bucket_prefix + \
'name=%s)' % self.extension_name
# Load the local store extension.
def conn_extensions(self, extlist):
# Windows doesn't support dynamically loaded extension libraries.
if os.name == 'nt':
extlist.skip_if_missing = True
extlist.extension('storage_sources', self.extension_name)
# Test sharing data between a primary and a secondary
def test_sharing(self):
# FIXME: WT-8235 Enable the test once file containing transaction ids is supported.
self.skipTest(
'Sharing the checkpoint file containing transaction ids is not supported'
)
ds = SimpleDataSet(self, self.uri, 10)
ds.populate()
ds.check()
self.session.checkpoint()
ds.check()
# Create a secondary database
dir2 = os.path.join(self.home, 'SECONDARY')
os.mkdir(dir2)
conn2 = self.setUpConnectionOpen(dir2)
session2 = conn2.open_session()
# Reference the tree from the secondary:
metac = self.session.open_cursor('metadata:')
metac2 = session2.open_cursor('metadata:', None, 'readonly=0')
uri2 = self.uri[:5] + '../' + self.uri[5:]
metac2[uri2] = metac[self.uri] + ",readonly=1"
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
newds = SimpleDataSet(self, self.uri, 10000)
newds.populate()
newds.check()
self.session.checkpoint()
newds.check()
# Check we can still read from the last checkpoint
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
# Bump to new checkpoint
origmeta = metac[self.uri]
checkpoint = re.search(r',checkpoint=\(.+?\)\)', origmeta).group(0)[1:]
self.pr('Orig checkpoint: ' + checkpoint)
session2.alter(uri2, checkpoint)
self.pr('New metadata on secondaery: ' + metac2[uri2])
# Check that we can see the new data
cursor2 = session2.open_cursor(uri2)
newds.check_cursor(cursor2)
class test_tiered03(wttest.WiredTigerTestCase):
K = 1024
M = 1024 * K
G = 1024 * M
# TODO: tiered: change this to a table: URI, otherwise we are
# not using tiered files. The use of a second directory for
# sharing would probably need to be reworked.
uri = 'file:test_tiered03'
storage_sources = [
('dirstore',
dict(auth_token=get_auth_token('dir_store'),
bucket=get_bucket1_name('dir_store'),
bucket_prefix="pfx_",
ss_name='dir_store')),
('s3',
dict(auth_token=get_auth_token('s3_store'),
bucket=get_bucket1_name('s3_store'),
bucket_prefix=generate_s3_prefix(),
ss_name='s3_store')),
]
# Occasionally add a lot of records to vary the amount of work flush does.
record_count_scenarios = wtscenario.quick_scenarios(
'nrecs', [10, 10000], [0.9, 0.1])
scenarios = wtscenario.make_scenarios(storage_sources,
record_count_scenarios,
prune=100,
prunelong=500)
absolute_bucket_dir = None # initialied in conn_config to an absolute path
def conn_config(self):
bucket_ret = self.bucket
# The bucket format for the S3 store is the name and the region separataed by a semi-colon.
if self.ss_name == 's3_store':
cache_dir = self.bucket[:self.bucket.find(';')] + '-cache'
else:
cache_dir = self.bucket + '-cache'
# We have multiple connections that want to share a bucket.
# For the directory store, the first time this function is called, we'll
# establish the absolute path for the bucket, and always use that for
# the bucket name.
# The cache directory name is a relative one, so it won't be shared
# between connections.
if self.ss_name == 'dir_store':
if self.absolute_bucket_dir == None:
self.absolute_bucket_dir = os.path.join(
os.getcwd(), self.bucket)
os.mkdir(self.absolute_bucket_dir)
bucket_ret = self.absolute_bucket_dir
return \
'debug_mode=(flush_checkpoint=true),' + \
'tiered_storage=(auth_token=%s,' % self.auth_token + \
'bucket=%s,' % bucket_ret + \
'cache_directory=%s,' % cache_dir + \
'bucket_prefix=%s,' % self.bucket_prefix + \
'name=%s)' % self.ss_name
# Load the storage store extension.
def conn_extensions(self, extlist):
config = ''
# S3 store is built as an optional loadable extension, not all test environments build S3.
if self.ss_name == 's3_store':
#config = '=(config=\"(verbose=1)\")'
extlist.skip_if_missing = True
#if self.ss_name == 'dir_store':
#config = '=(config=\"(verbose=1,delay_ms=200,force_delay=3)\")'
# Windows doesn't support dynamically loaded extension libraries.
if os.name == 'nt':
extlist.skip_if_missing = True
extlist.extension('storage_sources', self.ss_name + config)
# Test sharing data between a primary and a secondary
def test_sharing(self):
# FIXME: WT-8235 Enable the test once file containing transaction ids is supported.
self.skipTest(
'Sharing the checkpoint file containing transaction ids is not supported'
)
ds = SimpleDataSet(self, self.uri, 10)
ds.populate()
ds.check()
self.session.checkpoint()
ds.check()
# Create a secondary database
dir2 = os.path.join(self.home, 'SECONDARY')
os.mkdir(dir2)
conn2 = self.setUpConnectionOpen(dir2)
session2 = conn2.open_session()
# Reference the tree from the secondary:
metac = self.session.open_cursor('metadata:')
metac2 = session2.open_cursor('metadata:', None, 'readonly=0')
uri2 = self.uri[:5] + '../' + self.uri[5:]
metac2[uri2] = metac[self.uri] + ",readonly=1"
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
newds = SimpleDataSet(self, self.uri, 10000)
newds.populate()
newds.check()
self.session.checkpoint()
newds.check()
# Check we can still read from the last checkpoint
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
# Bump to new checkpoint
origmeta = metac[self.uri]
checkpoint = re.search(r',checkpoint=\(.+?\)\)', origmeta).group(0)[1:]
self.pr('Orig checkpoint: ' + checkpoint)
session2.alter(uri2, checkpoint)
self.pr('New metadata on secondaery: ' + metac2[uri2])
# Check that we can see the new data
cursor2 = session2.open_cursor(uri2)
newds.check_cursor(cursor2)