class test_create_excl(TieredConfigMixin, wttest.WiredTigerTestCase):
types = [
('file', dict(type='file:')),
('table', dict(type='table:')),
]
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, types)
def test_create_excl(self):
if self.is_tiered_scenario() and self.type == 'file:':
self.skipTest('Tiered storage does not support file URIs.')
uri = self.type + "create_excl"
# Create the object with the exclusive setting.
self.session.create(uri, "exclusive=true")
# Exclusive re-create should error.
self.assertRaises(wiredtiger.WiredTigerError,
lambda: self.session.create(uri, "exclusive=true"))
# Non-exclusive re-create is allowed.
self.session.create(uri, "exclusive=false")
# Exclusive create on a table that does not exist should succeed.
self.session.create(uri + "_non_existent", "exclusive=true")
# Non-exclusive create is allowed.
self.session.create(uri + "_non_existent1", "exclusive=false")
class test_schema07(TieredConfigMixin, wttest.WiredTigerTestCase):
tablename = 'table:test_schema07'
conn_config = 'cache_size=10MB'
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources)
@wttest.longtest("Creating many tables shouldn't fill the cache")
def test_many_tables(self):
s = self.session
# We have a 10MB cache, metadata is (well) over 512B per table,
# if we can create 20K tables, something must be cleaning up.
for i in range(20000):
uri = '%s-%06d' % (self.tablename, i)
s.create(uri)
c = s.open_cursor(uri)
# This will block if the metadata fills the cache
c["key"] = "value"
c.close()
self.dropUntilSuccess(self.session, uri)
class test_tiered02(wttest.WiredTigerTestCase, TieredConfigMixin):
complex_dataset = [
('simple_ds', dict(complex_dataset=False)),
('complex_ds', dict(complex_dataset=True)),
]
# Make scenarios for different cloud service providers
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered02',
tiered_only=True)
scenarios = make_scenarios(storage_sources, complex_dataset)
uri = "table:test_tiered02"
def conn_config(self):
return TieredConfigMixin.conn_config(self)
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
def progress(self, s):
self.verbose(3, s)
self.pr(s)
def confirm_flush(self, increase=True):
# Without directly using the filesystem API, directory listing is only supported on
# the directory store. Limit this check to the directory store.
if self.ss_name != 'dir_store':
return
got = sorted(list(os.listdir(self.bucket)))
self.pr('Flushed objects: ' + str(got))
if increase:
# WT-7639: we know that this assertion sometimes fails,
# we are collecting more data - we still want it to fail
# so it is noticed.
if len(got) <= self.flushed_objects:
from time import sleep
self.prout(
'directory items: {} is not greater than {}!'.format(
got, self.flushed_objects))
self.prout('waiting to see if it resolves')
for i in range(0, 10):
self.prout('checking again')
newgot = sorted(list(os.listdir(self.bucket)))
if len(newgot) > self.flushed_objects:
self.prout('resolved, now see: {}'.format(newgot))
break
sleep(i)
self.assertGreater(len(got), self.flushed_objects)
else:
self.assertEqual(len(got), self.flushed_objects)
self.flushed_objects = len(got)
def get_dataset(self, rows):
args = 'key_format=S'
if self.complex_dataset:
return ComplexDataSet(self, self.uri, rows, config=args)
else:
return SimpleDataSet(self, self.uri, rows, config=args)
# Test tiered storage with checkpoints and flush_tier calls.
def test_tiered(self):
self.flushed_objects = 0
self.pr("create sys")
self.progress('Create simple data set (10)')
ds = self.get_dataset(10)
self.progress('populate')
ds.populate()
ds.check()
self.progress('checkpoint')
self.session.checkpoint()
self.progress('flush_tier')
self.session.flush_tier(None)
self.confirm_flush()
ds.check()
self.close_conn()
self.progress('reopen_conn')
self.reopen_conn()
# Check what was there before.
ds = self.get_dataset(10)
ds.check()
self.progress('Create simple data set (50)')
ds = self.get_dataset(50)
self.progress('populate')
# Don't (re)create any of the tables or indices from here on out.
# We will keep a cursor open on the table, and creation requires
# exclusive access.
ds.populate(create=False)
ds.check()
self.progress('open extra cursor on ' + self.uri)
cursor = self.session.open_cursor(self.uri, None, None)
self.progress('checkpoint')
self.session.checkpoint()
self.progress('flush_tier')
self.session.flush_tier(None)
self.progress('flush_tier complete')
self.confirm_flush()
self.progress('Create simple data set (100)')
ds = self.get_dataset(100)
self.progress('populate')
ds.populate(create=False)
ds.check()
self.progress('checkpoint')
self.session.checkpoint()
self.progress('flush_tier')
self.session.flush_tier(None)
self.confirm_flush()
self.progress('Create simple data set (200)')
ds = self.get_dataset(200)
self.progress('populate')
ds.populate(create=False)
ds.check()
cursor.close()
self.progress('close_conn')
self.close_conn()
self.progress('reopen_conn')
self.reopen_conn()
# Check what was there before.
ds = self.get_dataset(200)
ds.check()
# Now add some more.
self.progress('Create simple data set (300)')
ds = self.get_dataset(300)
self.progress('populate')
ds.populate(create=False)
ds.check()
# We haven't done a flush so there should be
# nothing extra on the shared tier.
self.confirm_flush(increase=False)
self.progress('checkpoint')
self.session.checkpoint()
self.confirm_flush(increase=False)
self.progress('END TEST')
class test_tiered08(wttest.WiredTigerTestCase, TieredConfigMixin):
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered08',
tiered_only=True)
# Make scenarios for different cloud service providers
scenarios = make_scenarios(storage_sources)
batch_size = 100000
# Keep inserting keys until we've done this many flush and checkpoint ops.
ckpt_flush_target = 10
uri = "table:test_tiered08"
def conn_config(self):
return get_conn_config(self) + '),statistics=(fast)'
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
def get_stat(self, stat):
stat_cursor = self.session.open_cursor('statistics:')
val = stat_cursor[stat][2]
stat_cursor.close()
return val
def key_gen(self, i):
return 'KEY' + str(i)
def value_gen(self, i):
return 'VALUE_' + 'filler' * (i % 12) + str(i)
# Populate the test table. Keep adding keys until the desired number of flush and
# checkpoint operations have happened.
def populate(self):
ckpt_count = 0
flush_count = 0
nkeys = 0
self.pr('Populating tiered table')
c = self.session.open_cursor(self.uri, None, None)
while ckpt_count < self.ckpt_flush_target or flush_count < self.ckpt_flush_target:
for i in range(nkeys, nkeys + self.batch_size):
c[self.key_gen(i)] = self.value_gen(i)
nkeys += self.batch_size
ckpt_count = self.get_stat(stat.conn.txn_checkpoint)
flush_count = self.get_stat(stat.conn.flush_tier)
c.close()
return nkeys
def verify(self, key_count):
self.pr('Verifying tiered table')
c = self.session.open_cursor(self.uri, None, None)
for i in range(key_count):
self.assertEqual(c[self.key_gen(i)], self.value_gen(i))
c.close()
def test_tiered08(self):
# FIXME-WT-7833
# This test can trigger races in file handle access during flush_tier.
# We will re-enable it when that is fixed.
self.skipTest(
'Concurrent flush_tier and insert operations not supported yet.')
cfg = self.conn_config()
self.pr('Config is: ' + cfg)
intl_page = 'internal_page_max=16K'
base_create = 'key_format=S,value_format=S,' + intl_page
self.session.create(self.uri, base_create)
done = threading.Event()
ckpt = checkpoint_thread(self.conn, done)
flush = flush_tier_thread(self.conn, done)
# Start background threads and give them a chance to start.
ckpt.start()
flush.start()
time.sleep(0.5)
key_count = self.populate()
done.set()
flush.join()
ckpt.join()
self.verify(key_count)
self.close_conn()
self.pr('Reopening tiered table')
self.reopen_conn()
self.verify(key_count)
class test_tiered13(test_import_base, TieredConfigMixin):
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered13',
tiered_only=True)
# Make scenarios for different cloud service providers
scenarios = make_scenarios(storage_sources)
# If the 'uri' changes all the other names must change with it.
base = 'test_tiered13-000000000'
fileuri_base = 'file:' + base
file1uri = fileuri_base + '1.wtobj'
file2 = base + '2.wtobj'
file2uri = fileuri_base + '2.wtobj'
otherfile = 'other.wt'
otheruri = 'file:' + otherfile
uri = "table:test_tiered13"
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
def conn_config(self):
self.saved_conn = get_conn_config(self) + '),create'
return self.saved_conn
def test_tiered13(self):
# Create a new tiered table.
self.session.create(self.uri, 'key_format=S,value_format=S,')
# Add first data. Checkpoint, flush and close the connection.
c = self.session.open_cursor(self.uri)
c["0"] = "0"
c.close()
self.session.checkpoint()
self.session.flush_tier(None)
c = self.session.open_cursor(self.uri)
c["1"] = "1"
c.close()
self.session.checkpoint()
# We now have the second object existing, with data in it.
# Set up for the test.
# - Create the tiered table (above).
# - Find the metadata for the current file: object.
# - Set up a new database for importing.
#
# Testing import and tiered tables. All should error:
# - Try to import via the table:uri.
# - Try to import via the table:uri with the file object's metadata.
# - Try to import via the file:uri.
# - Try to import via the file:uri with the file object's metadata.
# - Try to import via a renamed file:name.wt.
# - Try to import via a renamed file:name.wt with the file object's metadata.
# Export the metadata for the current file object 2.
cursor = self.session.open_cursor('metadata:', None, None)
for k, v in cursor:
if k.startswith(self.file2uri):
fileobj_config = cursor[k]
if k.startswith(self.uri):
table_config = cursor[k]
cursor.close()
self.close_conn()
# Contruct the config strings.
import_enabled = 'import=(enabled,repair=true)'
import_meta = 'import=(enabled,repair=false,file_metadata=(' + \
fileobj_config + '))'
table_import_meta = table_config + ',import=(enabled,repair=false,file_metadata=(' + \
fileobj_config + '))'
# Set up the import database.
newdir = 'IMPORT_DB'
shutil.rmtree(newdir, ignore_errors=True)
os.mkdir(newdir)
newbucket = os.path.join(newdir, self.bucket)
if self.ss_name == 'dir_store':
os.mkdir(newbucket)
# It is tricky to work around the extension and connection bucket setup for
# creating the new import directory that is tiered-enabled.
ext = self.extensionsConfig()
conn_params = self.saved_conn + ext
self.conn = self.wiredtiger_open(newdir, conn_params)
self.session = self.setUpSessionOpen(self.conn)
# Copy the file to the file names we're going to test later.
self.copy_file(self.file2, '.', newdir)
copy_from = self.file2
copy_to = os.path.join(newdir, self.otherfile)
shutil.copy(copy_from, copy_to)
msg = '/Operation not supported/'
enoent = '/No such file/'
invalid = "/import for tiered storage is incompatible with the 'file_metadata' setting/"
# Try to import via the table:uri. This fails with ENOENT because
# it is looking for the normal on-disk file name. It cannot tell it
# is a tiered table in this case.
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create(self.uri, import_enabled), enoent)
# Try to import via the table:uri with file metadata.
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create(self.uri, table_import_meta), invalid)
# Try to import via the file:uri.
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create(self.file2uri, import_enabled), msg)
# Try to import via the file:uri with file metadata.
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create(self.file2uri, import_meta), invalid)
# Try to import via a renamed object. If we don't send in metadata,
# we cannot tell it was a tiered table until we read in the root page.
# Only test this in diagnostic mode which has an assertion.
#
# FIXME-8644 There is an error path bug in wt_bm_read preventing this from
# working correctly although the code to return an error is in the code.
# Uncomment these lines when that bug is fixed.
#if wiredtiger.diagnostic_build():
# self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
# lambda: self.session.create(self.otheruri, import_enabled), msg)
# Try to import via a renamed object with metadata.
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create(self.otheruri, import_meta), invalid)
class test_alter03(TieredConfigMixin, wttest.WiredTigerTestCase):
name = "alter03"
# Build all scenarios
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources)
def verify_metadata(self, table_metastr, lsm_metastr, file_metastr):
c = self.session.open_cursor('metadata:', None, None)
if table_metastr != '':
# We must find a table type entry for this object and its value
# should contain the provided table meta string.
c.set_key('table:' + self.name)
self.assertNotEqual(c.search(), wiredtiger.WT_NOTFOUND)
value = c.get_value()
self.assertTrue(value.find(table_metastr) != -1)
if lsm_metastr != '':
# We must find a lsm type entry for this object and its value
# should contain the provided lsm meta string.
c.set_key('lsm:' + self.name)
self.assertNotEqual(c.search(), wiredtiger.WT_NOTFOUND)
value = c.get_value()
self.assertTrue(value.find(lsm_metastr) != -1)
if file_metastr != '':
# We must find a file type entry for the object and its value
# should contain the provided file meta string.
if self.is_tiered_scenario():
c.set_key('file:' + self.name + '-0000000001.wtobj')
else:
c.set_key('file:' + self.name + '.wt')
self.assertNotEqual(c.search(), wiredtiger.WT_NOTFOUND)
value = c.get_value()
self.assertTrue(value.find(file_metastr) != -1)
c.close()
# Alter Table: Change the app_metadata and verify
def test_alter03_table_app_metadata(self):
uri = "table:" + self.name
entries = 100
create_params = 'key_format=i,value_format=i,'
app_meta_orig = 'app_metadata="meta_data_1",'
self.session.create(uri, create_params + app_meta_orig)
# Put some data in table.
c = self.session.open_cursor(uri, None)
for k in range(entries):
c[k + 1] = 1
c.close()
# Verify the string in the metadata
self.verify_metadata(app_meta_orig, '', app_meta_orig)
# Alter app metadata and verify
self.session.alter(uri, 'app_metadata="meta_data_2",')
self.verify_metadata('app_metadata="meta_data_2",', '',
'app_metadata="meta_data_2",')
# Alter app metadata, explicitly asking for exclusive access and verify
self.session.alter(
uri, 'app_metadata="meta_data_3",exclusive_refreshed=true,')
self.verify_metadata('app_metadata="meta_data_3",', '',
'app_metadata="meta_data_3",')
# Alter app metadata without taking exclusive lock and verify that only
# table object gets modified
self.session.alter(
uri, 'app_metadata="meta_data_4",exclusive_refreshed=false,')
self.verify_metadata('app_metadata="meta_data_4",', '',
'app_metadata="meta_data_3",')
# Open a cursor, insert some data and try to alter with session open.
# We should fail unless we ask not to take an exclusive lock
c2 = self.session.open_cursor(uri, None)
for k in range(entries):
c2[k + 1] = 2
self.assertRaisesException(
wiredtiger.WiredTigerError,
lambda: self.session.alter(uri, 'app_metadata="meta_data_5",'))
self.verify_metadata('app_metadata="meta_data_4",', '',
'app_metadata="meta_data_3",')
self.assertRaisesException(
wiredtiger.WiredTigerError, lambda: self.session.alter(
uri, 'exclusive_refreshed=true,app_metadata="meta_data_5",'))
self.verify_metadata('app_metadata="meta_data_4",', '',
'app_metadata="meta_data_3",')
self.session.alter(
uri, 'app_metadata="meta_data_5",exclusive_refreshed=false,')
self.verify_metadata('app_metadata="meta_data_5",', '',
'app_metadata="meta_data_3",')
c2.close()
# Close and reopen the connection.
# Confirm we retain the app_metadata as expected after reopen
self.reopen_conn()
self.verify_metadata('app_metadata="meta_data_5",', '',
'app_metadata="meta_data_3",')
# Alter LSM: A non exclusive alter should not be allowed
def test_alter03_lsm_app_metadata(self):
if self.is_tiered_scenario():
self.skipTest('Tiered storage does not support LSM.')
uri = "lsm:" + self.name
create_params = 'key_format=i,value_format=i,'
app_meta_orig = 'app_metadata="meta_data_1",'
self.session.create(uri, create_params + app_meta_orig)
# Try to alter app metadata without exclusive access and verify
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: self.session.alter(
uri, 'exclusive_refreshed=false,app_metadata="meta_data_2",'),
'/is applicable only on simple tables/')
self.verify_metadata('', 'app_metadata="meta_data_1",', '')
# Alter app metadata, explicitly asking for exclusive access and verify
self.session.alter(
uri, 'exclusive_refreshed=true,app_metadata="meta_data_2",')
self.verify_metadata('', 'app_metadata="meta_data_2",', '')
# Alter app metadata and verify
self.session.alter(uri, 'app_metadata="meta_data_3",')
self.verify_metadata('', 'app_metadata="meta_data_3",', '')
class test_tiered11(wttest.WiredTigerTestCase, TieredConfigMixin):
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered11',
tiered_only=True)
# Make scenarios for different cloud service providers
scenarios = make_scenarios(storage_sources)
# If the 'uri' changes all the other names must change with it.
base = 'test_tiered11-000000000'
nentries = 10
objuri = 'object:' + base + '1.wtobj'
tiereduri = "tiered:test_tiered11"
uri = "table:test_tiered11"
def conn_config(self):
self.saved_conn = get_conn_config(self) + ')'
return self.saved_conn
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
# Check for a specific string as part of the uri's metadata.
def check_metadata(self, uri, val_str, match=True):
#self.pr("Check_meta: uri: " + uri)
c = self.session.open_cursor('metadata:')
val = c[uri]
c.close()
#self.pr("Check_meta: metadata val: " + val)
if match:
#self.pr("Check_meta: Should see val_str: " + val_str)
self.assertTrue(val_str in val)
else:
#self.pr("Check_meta: Should not see val_str: " + val_str)
self.assertFalse(val_str in val)
def add_data(self, start):
c = self.session.open_cursor(self.uri)
# Begin by adding some data.
end = start + self.nentries
for i in range(start, end):
self.session.begin_transaction()
c[i] = i
# Jump the commit TS to leave rooom for the stable TS separate from any commit TS.
self.session.commit_transaction('commit_timestamp=' +
self.timestamp_str(i * 2))
# Set the oldest and stable timestamp to the end.
end_ts = self.timestamp_str(end - 1)
self.conn.set_timestamp('oldest_timestamp=' + end_ts +
',stable_timestamp=' + end_ts)
c.close()
return end_ts
# Test calling the flush_tier API.
def test_tiered11(self):
# Create a tiered table and checkpoint. Make sure the recorded
# timestamp is what we expect.
intl_page = 'internal_page_max=16K'
base_create = 'key_format=i,value_format=i,' + intl_page
self.session.create(self.uri, base_create)
end_ts = self.add_data(1)
self.session.checkpoint()
new_end_ts = self.add_data(self.nentries)
# We have a new stable timestamp, but after the checkpoint. Make
# sure the flush tier records the correct timestamp.
self.session.flush_tier(None)
# Make sure a new checkpoint doesn't change any of our timestamp info.
self.session.checkpoint()
flush_str = 'flush_timestamp="' + end_ts + '"'
self.check_metadata(self.tiereduri, flush_str)
self.check_metadata(self.objuri, flush_str)
# Make sure some flush time was saved. We don't know what it is other
# than it should not be zero.
time_str = "flush_time=0"
self.check_metadata(self.tiereduri, time_str, False)
self.check_metadata(self.objuri, time_str, False)
class test_tiered15(TieredConfigMixin, wttest.WiredTigerTestCase):
types = [
('file',
dict(type='file',
tiered_err=False,
non_tiered_err=False,
non_tiered_errmsg=None)),
('table',
dict(type='table',
tiered_err=True,
non_tiered_err=False,
non_tiered_errmsg=None)),
('tier',
dict(type='tier',
tiered_err=True,
non_tiered_err=False,
non_tiered_errmsg=None)),
('tiered',
dict(type='tiered',
tiered_err=True,
non_tiered_err=True,
non_tiered_errmsg="/Invalid argument/")),
('colgroup',
dict(type='colgroup',
tiered_err=True,
non_tiered_err=True,
non_tiered_errmsg=None)),
('index',
dict(type='index',
tiered_err=True,
non_tiered_err=True,
non_tiered_errmsg="/Invalid argument/")),
('lsm',
dict(type='lsm',
tiered_err=True,
non_tiered_err=False,
non_tiered_errmsg=None)),
('backup',
dict(type='backup',
tiered_err=True,
non_tiered_err=False,
non_tiered_errmsg="/Operation not supported/")),
]
# This is different to the self.is_tiered_scenario() value - that one configures tiered
# storage at a connection level but here we want to configure at the table level.
is_tiered_table = [
('tiered_table', dict(is_tiered_table=True)),
('nontiered_table', dict(is_tiered_table=False)),
]
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, types)
def test_create_type_config(self):
if not self.is_tiered_scenario():
self.skipTest(
'The test should only test connections configured with tiered storage.'
)
tiered_status = "tiered" if self.is_tiered_table else "nontiered"
uri = "table:" + tiered_status + self.type
if self.is_tiered_table:
# Creating a tiered table with the type configuration set to file should succeed, and all other types
# should not succeed.
if self.type == "file":
self.session.create(uri, "type=" + self.type)
else:
tiered_errmsg = "/Operation not supported/"
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create(uri, "type=" + self.type),
tiered_errmsg)
else:
# Creating a non-tiered table within a connection configured with tiered storage should allow the type
# configuration to be set to some other types besides "file".
if not self.non_tiered_err:
self.session.create(
uri, "tiered_storage=(name=none),type=" + self.type)
else:
# The types "tiered", "colgroup", "index", and "backup" are not supported when creating a non-tiered
# table in a non-tiered connection. It is expected these types will also not work with non-tiered
# tables in a connection configured with tiered storage.
# Additionally, configuring type to "colgroup" causes WiredTiger to crash, skip this scenario.
if self.type == "colgroup":
self.skipTest(
'Skip the colgroup type configuration as we expect it to crash.'
)
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: self.session.create(
uri, "tiered_storage=(name=none),type=" + self.type),
self.non_tiered_errmsg)
class test_alter02(TieredConfigMixin, wttest.WiredTigerTestCase):
entries = 500
# Binary values.
value = u'\u0001\u0002abcd\u0003\u0004'
value2 = u'\u0001\u0002dcba\u0003\u0004'
conn_log = [
('conn-always-logged', dict(conncreate=True, connreopen=True)),
('conn-create-logged', dict(conncreate=True, connreopen=False)),
('conn-reopen-logged', dict(conncreate=False, connreopen=True)),
('conn-never-logged', dict(conncreate=False, connreopen=False)),
]
types = [
('file', dict(uri='file:', use_cg=False, use_index=False)),
('lsm', dict(uri='lsm:', use_cg=False, use_index=False)),
('table-cg', dict(uri='table:', use_cg=True, use_index=False)),
('table-index', dict(uri='table:', use_cg=False, use_index=True)),
('table-simple', dict(uri='table:', use_cg=False, use_index=False)),
]
tables = [
('always-logged', dict(name='table0', logcreate=True, logalter=True)),
('create-logged', dict(name='table1', logcreate=True, logalter=False)),
('alter-logged', dict(name='table2', logcreate=False, logalter=True)),
('never-logged', dict(name='table3', logcreate=False, logalter=False)),
]
reopen = [
('no-reopen', dict(reopen=False)),
('reopen', dict(reopen=True)),
]
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, conn_log, types, tables,
reopen)
# This test varies the log setting. Override the standard methods.
def setUpConnectionOpen(self, dir):
return None
def setUpSessionOpen(self, conn):
return None
def ConnectionOpen(self):
self.home = '.'
tiered_config = self.conn_config()
tiered_config += self.extensionsConfig()
# In case the open starts additional threads, flush first to avoid confusion.
sys.stdout.flush()
conn_params = 'create,log=(file_max=100K,remove=false,%s)' % self.uselog
if tiered_config != '':
conn_params += ',' + tiered_config
try:
self.conn = wiredtiger.wiredtiger_open(self.home, conn_params)
except wiredtiger.WiredTigerError as e:
print("Failed conn at '%s' with config '%s'" % (dir, conn_params))
self.session = self.conn.open_session()
# Verify the metadata string for this URI and that its setting in the
# metadata file is correct.
def verify_metadata(self, metastr):
if metastr == '':
return
cursor = self.session.open_cursor('metadata:', None, None)
#
# Walk through all the metadata looking for the entries that are
# the file URIs for components of the table.
#
found = False
while True:
ret = cursor.next()
if ret != 0:
break
key = cursor.get_key()
check_meta = ((key.find("lsm:") != -1 or key.find("file:") != -1) \
and key.find(self.name) != -1)
if check_meta:
value = cursor[key]
found = True
self.assertTrue(value.find(metastr) != -1)
cursor.close()
self.assertTrue(found == True)
# Verify the data in the log. If the data should be logged we write one
# value. If it should not be logged, we write a different value.
def verify_logrecs(self, expected_keys):
c = self.session.open_cursor('log:', None, None)
count = 0
while c.next() == 0:
# lsn.file, lsn.offset, opcount
keys = c.get_key()
# txnid, rectype, optype, fileid, logrec_key, logrec_value
values = c.get_value()
if self.value.encode() in values[5]: # logrec_value
count += 1
self.assertFalse(self.value2.encode() in values[5])
c.close()
#
# We check that we saw the expected keys at twice the rate because
# the log cursor, for each commit record, will first return the entire,
# full record, and then return the individual operation. We will detect
# the string in both records.
self.assertEqual(count, expected_keys * 2)
# Alter: Change the log setting after creation
def test_alter02_log(self):
if self.is_tiered_scenario() and (self.uri == 'lsm:'
or self.uri == 'file:'):
self.skipTest('Tiered storage does not support LSM or file URIs.')
uri = self.uri + self.name
create_params = 'key_format=i,value_format=S,'
complex_params = ''
# Set up logging for the connection.
if self.conncreate:
self.uselog = 'enabled=true'
conn_logged = 1
else:
self.uselog = 'enabled=false'
conn_logged = 0
self.ConnectionOpen()
# Set up logging for the table.
if self.logcreate:
log_param = 'log=(enabled=true)'
table_logged = 1
else:
log_param = 'log=(enabled=false)'
table_logged = 0
create_params += '%s,' % log_param
complex_params += '%s,' % log_param
cgparam = ''
if self.use_cg or self.use_index:
cgparam = 'columns=(k,v),'
if self.use_cg:
cgparam += 'colgroups=(g0),'
self.session.create(uri, create_params + cgparam)
# Add in column group or index settings.
if self.use_cg:
cgparam = 'columns=(v),'
suburi = 'colgroup:' + self.name + ':g0'
self.session.create(suburi, complex_params + cgparam)
if self.use_index:
suburi = 'index:' + self.name + ':i0'
self.session.create(suburi, complex_params + cgparam)
# Put some data in table.
c = self.session.open_cursor(uri, None)
if self.logcreate:
myvalue = self.value
else:
myvalue = self.value2
for k in range(self.entries):
c[k] = myvalue
c.close()
# Verify the logging string in the metadata.
self.verify_metadata(log_param)
# Verify the logged operations only if logging is enabled.
expected_keys = conn_logged * table_logged * self.entries
if conn_logged != 0:
self.pr("EXPECTED KEYS 1: " + str(expected_keys))
self.verify_logrecs(expected_keys)
# Set the alter setting for the table.
if self.logalter:
log_str = 'log=(enabled=true)'
table_logged = 1
else:
log_str = 'log=(enabled=false)'
table_logged = 0
alter_param = '%s' % log_str
special = self.use_cg or self.use_index
# Set the log setting on the new connection.
if self.reopen:
if self.connreopen:
self.uselog = 'enabled=true'
conn_logged = 1
else:
self.uselog = 'enabled=false'
conn_logged = 0
self.conn.close()
self.ConnectionOpen()
self.session.alter(uri, alter_param)
if special:
self.session.alter(suburi, alter_param)
self.verify_metadata(log_str)
# Put some more data in table.
c = self.session.open_cursor(uri, None)
if self.logalter:
myvalue = self.value
else:
myvalue = self.value2
for k in range(self.entries):
c[k + self.entries] = myvalue
c.close()
# If we logged the new connection and the table, add in the
# number of keys we expect.
expected_keys += conn_logged * table_logged * self.entries
# if self.logalter and self.connreopen == self.reopen:
# expected_keys += self.entries
# If we logged the connection at any time then check
# the log records.
if self.conncreate or (self.connreopen and self.reopen):
self.pr("EXPECTED KEYS 2: " + str(expected_keys))
self.verify_logrecs(expected_keys)
class test_tiered04(wttest.WiredTigerTestCase, TieredConfigMixin):
# Make scenarios for different cloud service providers
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered04',
tiered_only=True)
scenarios = make_scenarios(storage_sources)
# If the 'uri' changes all the other names must change with it.
base = 'test_tiered04-000000000'
fileuri_base = 'file:' + base
obj1file = base + '1.wtobj'
obj2file = base + '2.wtobj'
objuri = 'object:' + base + '1.wtobj'
tiereduri = "tiered:test_tiered04"
tieruri = "tier:test_tiered04"
uri = "table:test_tiered04"
uri1 = "table:test_other_tiered04"
uri_none = "table:test_local04"
file_none = "file:test_local04.wt"
retention = 3
retention1 = 600
def conn_config(self):
if self.ss_name == 'dir_store':
os.mkdir(self.bucket)
os.mkdir(self.bucket1)
self.saved_conn = get_conn_config(self) + 'local_retention=%d)'\
% self.retention
return self.saved_conn
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
# Check for a specific string as part of the uri's metadata.
def check_metadata(self, uri, val_str):
c = self.session.open_cursor('metadata:')
val = c[uri]
c.close()
self.assertTrue(val_str in val)
def get_stat(self, stat, uri):
if uri == None:
stat_cursor = self.session.open_cursor('statistics:')
else:
stat_cursor = self.session.open_cursor('statistics:' + uri)
val = stat_cursor[stat][2]
stat_cursor.close()
return val
def check(self, tc, base, n):
get_check(self, tc, base, n)
# Test calling the flush_tier API.
def test_tiered(self):
# Create three tables. One using the system tiered storage, one
# specifying its own bucket and object size and one using no
# tiered storage. Use stats to verify correct setup.
intl_page = 'internal_page_max=16K'
base_create = 'key_format=S,value_format=S,' + intl_page
self.pr("create sys")
self.session.create(self.uri, base_create)
conf = \
',tiered_storage=(auth_token=%s,' % self.auth_token + \
'bucket=%s,' % self.bucket1 + \
'bucket_prefix=%s,' % self.bucket_prefix1 + \
'local_retention=%d,' % self.retention1 + \
'name=%s)' % self.ss_name
self.pr("create non-sys tiered")
self.session.create(self.uri1, base_create + conf)
conf = ',tiered_storage=(name=none)'
self.pr("create non tiered/local")
self.session.create(self.uri_none, base_create + conf)
c = self.session.open_cursor(self.uri)
c1 = self.session.open_cursor(self.uri1)
cn = self.session.open_cursor(self.uri_none)
c["0"] = "0"
c1["0"] = "0"
cn["0"] = "0"
self.check(c, 0, 1)
self.check(c1, 0, 1)
self.check(cn, 0, 1)
c.close()
flush = 0
# Check the local retention. After a flush_tier call the object file should exist in
# the local database. Then after sleeping long enough it should be removed.
self.pr("flush tier no checkpoint")
self.session.flush_tier(None)
flush += 1
# We should not have flushed either tiered table.
skip = self.get_stat(stat.conn.flush_tier_skipped, None)
self.assertEqual(skip, 2)
self.session.checkpoint()
self.session.flush_tier(None)
# Now we should have switched both tables. The skip value should stay the same.
skip = self.get_stat(stat.conn.flush_tier_skipped, None)
self.assertEqual(skip, 2)
switch = self.get_stat(stat.conn.flush_tier_switched, None)
self.assertEqual(switch, 2)
flush += 1
self.pr("Check for ")
self.pr(self.obj1file)
self.assertTrue(os.path.exists(self.obj1file))
self.assertTrue(os.path.exists(self.obj2file))
remove1 = self.get_stat(stat.conn.local_objects_removed, None)
time.sleep(self.retention + 1)
# We call flush_tier here because otherwise the internal thread that
# processes the work units won't run for a while. This call will signal
# the internal thread to process the work units.
self.session.flush_tier('force=true')
flush += 1
# We still sleep to give the internal thread a chance to run. Some slower
# systems can fail here if we don't give them time.
time.sleep(1)
self.pr("Check removal of ")
self.pr(self.obj1file)
self.assertFalse(os.path.exists(self.obj1file))
remove2 = self.get_stat(stat.conn.local_objects_removed, None)
self.assertTrue(remove2 > remove1)
c = self.session.open_cursor(self.uri)
c["1"] = "1"
c1["1"] = "1"
cn["1"] = "1"
self.check(c, 0, 2)
c.close()
c = self.session.open_cursor(self.uri)
c["2"] = "2"
c1["2"] = "2"
cn["2"] = "2"
self.check(c, 0, 3)
c1.close()
cn.close()
self.session.checkpoint()
self.pr("flush tier again, holding open cursor")
self.session.flush_tier(None)
flush += 1
c["3"] = "3"
self.check(c, 0, 4)
c.close()
calls = self.get_stat(stat.conn.flush_tier, None)
self.assertEqual(calls, flush)
# As we flush each object, the next object exists, but our first flush was a no-op.
# So the value for the last file: object should be 'flush'.
last = 'last=' + str(flush)
# For now all earlier objects exist. So it is always 1 until garbage collection
# starts removing them.
oldest = 'oldest=1'
fileuri = self.fileuri_base + str(flush) + '.wtobj'
self.check_metadata(self.tiereduri, intl_page)
self.check_metadata(self.tiereduri, last)
self.check_metadata(self.tiereduri, oldest)
self.check_metadata(fileuri, intl_page)
self.check_metadata(self.objuri, intl_page)
# Check for the correct tiered_object setting for both tiered and not tiered tables.
tiered_false = 'tiered_object=false'
tiered_true = 'tiered_object=true'
self.check_metadata(fileuri, tiered_true)
self.check_metadata(self.objuri, tiered_true)
self.check_metadata(self.tieruri, tiered_true)
self.check_metadata(self.file_none, tiered_false)
# Now test some connection statistics with operations.
retain = self.get_stat(stat.conn.tiered_retention, None)
self.assertEqual(retain, self.retention)
self.session.flush_tier(None)
skip1 = self.get_stat(stat.conn.flush_tier_skipped, None)
switch1 = self.get_stat(stat.conn.flush_tier_switched, None)
# Make sure the last checkpoint and this flush tier are timed differently
# so that we can specifically check the statistics and code paths in the test.
# Sleep some to control the execution.
time.sleep(2)
self.session.flush_tier('force=true')
skip2 = self.get_stat(stat.conn.flush_tier_skipped, None)
switch2 = self.get_stat(stat.conn.flush_tier_switched, None)
self.assertGreater(switch2, switch1)
self.assertEqual(skip1, skip2)
flush += 2
calls = self.get_stat(stat.conn.flush_tier, None)
self.assertEqual(calls, flush)
# Test reconfiguration.
config = 'tiered_storage=(local_retention=%d)' % self.retention1
self.pr("reconfigure")
self.conn.reconfigure(config)
retain = self.get_stat(stat.conn.tiered_retention, None)
self.assertEqual(retain, self.retention1)
# Call flush_tier with its various configuration arguments. It is difficult
# to force a timeout or lock contention with a unit test. So just test the
# call for now.
#
# There have been no data changes nor checkpoints since the last flush_tier with
# force, above. The skip statistics should increase and the switched
# statistics should stay the same.
skip1 = self.get_stat(stat.conn.flush_tier_skipped, None)
switch1 = self.get_stat(stat.conn.flush_tier_switched, None)
self.session.flush_tier('timeout=100')
skip2 = self.get_stat(stat.conn.flush_tier_skipped, None)
switch2 = self.get_stat(stat.conn.flush_tier_switched, None)
self.assertEqual(switch1, switch2)
self.assertGreater(skip2, skip1)
self.session.flush_tier('lock_wait=false')
self.session.flush_tier('sync=off')
flush += 3
self.pr("reconfigure get stat")
calls = self.get_stat(stat.conn.flush_tier, None)
self.assertEqual(calls, flush)
# Test that the checkpoint and flush times work across a connection restart.
# Make modifications and then close the connection (which will checkpoint).
# Reopen the connection and call flush_tier. Verify this flushes the object.
c = self.session.open_cursor(self.uri)
c["4"] = "4"
self.check(c, 0, 5)
c.close()
# Manually reopen the connection because the default function above tries to
# make the bucket directories.
self.reopen_conn(config=self.saved_conn)
remove1 = self.get_stat(stat.conn.local_objects_removed, None)
skip1 = self.get_stat(stat.conn.flush_tier_skipped, None)
switch1 = self.get_stat(stat.conn.flush_tier_switched, None)
self.session.flush_tier(None)
skip2 = self.get_stat(stat.conn.flush_tier_skipped, None)
switch2 = self.get_stat(stat.conn.flush_tier_switched, None)
# The first flush_tier after restart should have queued removal work units
# for other objects. Sleep and then force a flush tier to signal the internal
# thread and make sure that some objects were removed.
time.sleep(self.retention + 1)
self.session.flush_tier('force=true')
# Sleep to give the internal thread time to run and process.
time.sleep(1)
self.assertFalse(os.path.exists(self.obj1file))
remove2 = self.get_stat(stat.conn.local_objects_removed, None)
self.assertTrue(remove2 > remove1)
#
# Due to the above modification, we should skip the 'other' table while
# switching the main tiered table. Therefore, both the skip and switch
# values should increase by one.
self.assertEqual(skip2, skip1 + 1)
self.assertEqual(switch2, switch1 + 1)
class test_tiered12(wttest.WiredTigerTestCase, TieredConfigMixin):
# Make scenarios for different cloud service providers
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered12',
tiered_only=True)
scenarios = make_scenarios(storage_sources)
# If the 'uri' changes all the other names must change with it.
base = 'test_tiered12-000000000'
obj1file = base + '1.wtobj'
uri = "table:test_tiered12"
retention = 1
saved_conn = ''
def conn_config(self):
self.saved_conn = get_conn_config(self) + 'local_retention=%d),' \
% self.retention + 'timing_stress_for_test=(tiered_flush_finish)'
return self.saved_conn
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
def check(self, tc, base, n):
get_check(self, tc, base, n)
def test_tiered(self):
# Default cache location is cache-<bucket-name>
cache = "cache-" + self.bucket
# The bucket format for the S3 store is the name and the region separataed by a semi-colon.
# Strip off the region to get the cache folder.
if self.ss_name == 's3_store':
cache = cache[:cache.find(';')]
# Create a table. Add some data. Checkpoint and flush tier.
# We have configured the timing stress for tiered caching which delays
# the internal thread calling flush_finish for 1 second.
# So after flush tier completes, check that the cached object does not
# exist. Then sleep and check that it does exist.
#
# The idea is to make sure flush_tier is not waiting for unnecessary work
# to be done, but returns as soon as the copying to shared storage completes.
self.session.create(self.uri, 'key_format=S,value_format=S,')
# Add data. Checkpoint and flush.
c = self.session.open_cursor(self.uri)
c["0"] = "0"
self.check(c, 0, 1)
c.close()
self.session.checkpoint()
self.session.flush_tier(None)
# On directory store, the bucket object should exist.
if self.ss_name == 'dir_store':
bucket_obj = os.path.join(self.bucket,
self.bucket_prefix + self.obj1file)
self.assertTrue(os.path.exists(bucket_obj))
# Sleep more than the one second stress timing amount and give the thread time to run.
time.sleep(2)
# After sleeping, the internal thread should have created the cached object.
cache_obj = os.path.join(cache, self.bucket_prefix + self.obj1file)
self.assertTrue(os.path.exists(cache_obj))
class test_schema04(TieredConfigMixin, wttest.WiredTigerTestCase):
"""
Test indices with duplicates.
Our set of rows looks like a multiplication table:
row 0: [ 0, 0, 0, 0, 0, 0 ]
row 1: [ 0, 1, 2, 3, 4, 5 ]
row 2: [ 0, 2, 4, 6, 8, 10 ]
with the twist that entries are mod 100. So, looking further:
row 31: [ 0, 31, 62, 93, 24, 55 ]
Each column is placed into its own index. The mod twist,
as well as the 0th column, guarantees we'll have some duplicates.
"""
nentries = 100
index = [
('index-before', {
'create_index': 0
}),
('index-during', {
'create_index': 1
}),
('index-after', {
'create_index': 2
}),
]
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, index)
def create_indices(self):
# Create 6 index files, each with a column from the main table
for i in range(0, 6):
self.session.create("index:schema04:x" + str(i),
"key_format=i,columns=(v" + str(i) + "),")
# We split the population into two phases
# (in anticipation of future tests that create
# indices between the two population steps).
def populate(self, phase):
cursor = self.session.open_cursor('table:schema04', None, None)
if phase == 0:
range_from = 0
range_to = self.nentries // 2
else:
range_from = self.nentries // 2
range_to = self.nentries
for i in range(range_from, range_to):
# e.g. element 31 is '0,31,62,93,24,55'
cursor.set_key(i)
cursor.set_value((i * 0) % 100, (i * 1) % 100, (i * 2) % 100,
(i * 3) % 100, (i * 4) % 100, (i * 5) % 100)
cursor.insert()
cursor.close()
def check_entries(self):
cursor = self.session.open_cursor('table:schema04', None, None)
icursor = []
for i in range(0, 6):
icursor.append(
self.session.open_cursor('index:schema04:x' + str(i), None,
None))
i = 0
for kv in cursor:
# Check main table
expect = [(i * j) % 100 for j in range(0, 6)]
primkey = kv.pop(0)
self.assertEqual(i, primkey)
self.assertEqual(kv, expect)
for j in range(0, 6):
self.assertEqual((i * j) % 100, kv[j])
for idx in range(0, 6):
c = icursor[idx]
indexkey = (i * idx) % 100
c.set_key(indexkey)
self.assertEqual(c.search(), 0)
value = c.get_value()
while value != expect and value[idx] == expect[idx]:
c.next()
value = c.get_value()
self.assertEqual(value, expect)
i += 1
self.assertEqual(self.nentries, i)
def test_index(self):
self.session.create(
"table:schema04", "key_format=i,value_format=iiiiii,"
"columns=(primarykey,v0,v1,v2,v3,v4,v5)")
if self.create_index == 0:
self.create_indices()
self.populate(0)
if self.create_index == 1:
self.create_indices()
self.populate(1)
if self.create_index == 2:
self.create_indices()
self.check_entries()
class test_tiered16(TieredConfigMixin, wttest.WiredTigerTestCase):
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources)
def check_cache(self, cache_dir, expect1):
got = sorted(list(os.listdir(cache_dir)))
expect = sorted(expect1)
self.assertEquals(got, expect)
def check_bucket(self, expect1):
got = sorted(list(os.listdir(self.bucket)))
expect = sorted(expect1)
self.assertEquals(got, expect)
def test_remove_shared(self):
uri_a = "table:tiereda"
uri_b = "table:tieredb"
base_b = "tieredb-000000000"
obj1file_b = base_b + "1.wtobj"
obj2file_b = base_b + "2.wtobj"
self.session.create(uri_a, "key_format=S,value_format=S")
self.session.create(uri_b, "key_format=S,value_format=S")
# It is invalid for the user to attempt to force removal of shared files
# if they have configured for underlying files to not be removed.
if self.is_tiered_scenario():
msg = '/drop for tiered storage object must configure removal of underlying files/'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: self.session.drop(
uri_a, "remove_files=false,remove_shared=true"), msg)
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: self.session.drop(
uri_a, "force=true,remove_files=false,remove_shared=true"),
msg)
# Currently we are only running the test with dir_store because the remove_shared configuration
# is not yet implemented for the S3 storage source.
if self.is_tiered_scenario() and self.ss_name == 'dir_store':
# If a cache directory is not provided, the default cache directory is "cache-" appended to
# the bucket directory.
cache_dir = "cache-" + self.bucket
c = self.session.open_cursor(uri_a)
c["a"] = "a"
c["b"] = "b"
c.close()
self.session.checkpoint()
self.session.flush_tier(None)
c2 = self.session.open_cursor(uri_b)
c2["c"] = "c"
c2["d"] = "d"
c2.close()
self.session.checkpoint()
self.session.flush_tier(None)
self.session.drop(uri_a, "remove_files=true,remove_shared=true")
# The shared object files corresponding to the first table should have been removed from
# both the bucket and cache directories but the shared object files corresponding to the
# second table should still remain.
self.check_cache(cache_dir, [
self.bucket_prefix + obj1file_b,
self.bucket_prefix + obj2file_b
])
self.check_bucket([
self.bucket_prefix + obj1file_b,
self.bucket_prefix + obj2file_b
])
self.session.drop(uri_b, "remove_files=true,remove_shared=true")
# The shared object files corresponding to the second table should have been removed.
self.check_cache(cache_dir, [])
self.check_bucket([])
class test_schema06(TieredConfigMixin, wttest.WiredTigerTestCase):
"""
Test basic operations
"""
nentries = 1000
types = [
('normal', {
'type': 'normal',
'idx_config': ''
}),
('lsm', {
'type': 'lsm',
'idx_config': ',type=lsm'
}),
]
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, types)
def flip(self, inum, val):
"""
Defines a unique transformation of values for each index number.
We reverse digits so the generated values are not perfectly ordered.
"""
newval = str((inum + 1) * val)
return newval[::-1] # reversed digits
def unflip(self, inum, flipped):
"""
The inverse of flip.
"""
newval = flipped[::-1]
return int(newval) / (inum + 1)
def create_index(self, inum):
colname = "s" + str(inum)
self.session.create("index:schema06:" + colname,
"columns=(" + colname + ")" + self.idx_config)
def drop_index(self, inum):
colname = "s" + str(inum)
self.dropUntilSuccess(self.session, "index:schema06:" + colname)
def test_index_stress(self):
if self.is_tiered_scenario() and self.type == 'lsm':
self.skipTest('Tiered storage does not support LSM URIs.')
self.session.create(
"table:schema06", "key_format=S,value_format=SSSSSS," +
"columns=(key,s0,s1,s2,s3,s4,s5),colgroups=(c1,c2)")
self.create_index(0)
self.session.create("colgroup:schema06:c1", "columns=(s0,s1,s4)")
self.create_index(1)
self.session.create("colgroup:schema06:c2", "columns=(s2,s3,s5)")
cursor = self.session.open_cursor('table:schema06', None, None)
for i in range(0, self.nentries):
cursor.set_key(self.flip(0, i))
values = [self.flip(inum, i) for inum in range(6)]
cursor.set_value(values[0], values[1], values[2], values[3],
values[4], values[5])
cursor.insert()
cursor.close()
self.drop_index(0)
self.drop_index(1)
def check_entries(self, check_indices):
cursor = self.session.open_cursor('table:main', None, None)
# spot check via search
n = self.nentries
for i in (n // 5, 0, n - 1, n - 2, 1):
cursor.set_key(i, 'key' + str(i))
square = i * i
cube = square * i
cursor.search()
(s1, i2, s3, i4) = cursor.get_values()
self.assertEqual(s1, 'val' + str(square))
self.assertEqual(i2, square)
self.assertEqual(s3, 'val' + str(cube))
self.assertEqual(i4, cube)
count = 0
# then check all via cursor
cursor.reset()
for ikey, skey, s1, i2, s3, i4 in cursor:
i = ikey
square = i * i
cube = square * i
self.assertEqual(ikey, i)
self.assertEqual(skey, 'key' + str(i))
self.assertEqual(s1, 'val' + str(square))
self.assertEqual(i2, square)
self.assertEqual(s3, 'val' + str(cube))
self.assertEqual(i4, cube)
count += 1
cursor.close()
self.assertEqual(count, n)
if check_indices:
# we check an index that was created before populating
cursor = self.session.open_cursor('index:main:S1i4', None, None)
count = 0
for s1key, i4key, s1, i2, s3, i4 in cursor:
i = int(i4key**(1 // 3.0) + 0.0001) # cuberoot
self.assertEqual(s1key, s1)
self.assertEqual(i4key, i4)
ikey = i
skey = 'key' + str(i)
square = i * i
cube = square * i
self.assertEqual(ikey, i)
self.assertEqual(skey, 'key' + str(i))
self.assertEqual(s1, 'val' + str(square))
self.assertEqual(i2, square)
self.assertEqual(s3, 'val' + str(cube))
self.assertEqual(i4, cube)
count += 1
cursor.close()
self.assertEqual(count, n)
# we check an index that was created after populating
cursor = self.session.open_cursor('index:main:i2S1i4', None, None)
count = 0
for i2key, s1key, i4key, s1, i2, s3, i4 in cursor:
i = int(i4key**(1 // 3.0) + 0.0001) # cuberoot
self.assertEqual(i2key, i2)
self.assertEqual(s1key, s1)
self.assertEqual(i4key, i4)
ikey = i
skey = 'key' + str(i)
square = i * i
cube = square * i
self.assertEqual(ikey, i)
self.assertEqual(skey, 'key' + str(i))
self.assertEqual(s1, 'val' + str(square))
self.assertEqual(i2, square)
self.assertEqual(s3, 'val' + str(cube))
self.assertEqual(i4, cube)
count += 1
cursor.close()
self.assertEqual(count, n)
class test_tiered06(wttest.WiredTigerTestCase, TieredConfigMixin):
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered06',
tiered_only=True)
# Make scenarios for different cloud service providers
scenarios = make_scenarios(storage_sources)
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
def breakpoint(self):
import pdb, sys
sys.stdin = open('/dev/tty', 'r')
sys.stdout = open('/dev/tty', 'w')
sys.stderr = open('/dev/tty', 'w')
pdb.set_trace()
def get_storage_source(self):
return self.conn.get_storage_source(self.ss_name)
def get_fs_config(self, prefix='', cache_dir=''):
conf = ''
if prefix:
conf += ',prefix=' + prefix
if cache_dir:
conf += ',cache_directory=' + cache_dir
return conf
def test_ss_basic(self):
# Test some basic functionality of the storage source API, calling
# each supported method in the API at least once.
session = self.session
ss = self.get_storage_source()
# Since this class has multiple tests, append test name to the prefix to
# avoid namespace collison. 0th element on the stack is the current function.
prefix = self.bucket_prefix + inspect.stack()[0][3] + '/'
# The directory store needs the bucket created as a directory on the filesystem.
if self.ss_name == 'dir_store':
os.mkdir(self.bucket)
fs = ss.ss_customize_file_system(session, self.bucket, self.auth_token,
self.get_fs_config(prefix))
# The object doesn't exist yet.
self.assertFalse(fs.fs_exist(session, 'foobar'))
# We cannot use the file system to create files, it is readonly.
# So use python I/O to build up the file.
f = open('foobar', 'wb')
# The object still doesn't exist yet.
self.assertFalse(fs.fs_exist(session, 'foobar'))
outbytes = ('MORE THAN ENOUGH DATA\n' * 100000).encode()
f.write(outbytes)
f.close()
# Nothing is in the directory list until a flush.
self.assertEquals(fs.fs_directory_list(session, '', ''), [])
# Flushing copies the file into the file system.
ss.ss_flush(session, fs, 'foobar', 'foobar', None)
ss.ss_flush_finish(session, fs, 'foobar', 'foobar', None)
# The object exists now.
self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar'])
self.assertTrue(fs.fs_exist(session, 'foobar'))
fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data,
FileSystem.open_readonly)
inbytes = bytes(1000000) # An empty buffer with a million zero bytes.
fh.fh_read(session, 0, inbytes) # Read into the buffer.
self.assertEquals(outbytes[0:1000000], inbytes)
self.assertEquals(fs.fs_size(session, 'foobar'), len(outbytes))
self.assertEquals(fh.fh_size(session), len(outbytes))
fh.close(session)
# The fh_lock call doesn't do anything in the directory and S3 store implementation.
fh = fs.fs_open_file(session, 'foobar', FileSystem.open_file_type_data,
FileSystem.open_readonly)
fh.fh_lock(session, True)
fh.fh_lock(session, False)
fh.close(session)
# Attempt to open a file with incorrect name.
if self.ss_name == 's3_store':
msg = 'No such file or directory'
with self.expectedStderrPattern('.*request to S3 failed.*'):
self.assertRaisesException(
wiredtiger.WiredTigerError, lambda: fs.fs_open_file(
session, 'foobar2', FileSystem.open_file_type_data,
FileSystem.open_readonly), msg)
# Files that have been flushed cannot be manipulated.
with self.expectedStderrPattern(
'foobar: rename of file not supported'):
self.assertRaisesException(
wiredtiger.WiredTigerError,
lambda: fs.fs_rename(session, 'foobar', 'barfoo', 0))
self.assertEquals(fs.fs_directory_list(session, '', ''), ['foobar'])
if self.ss_name == 's3_store':
# Files that have been flushed cannot be manipulated through the custom file system.
with self.expectedStderrPattern(
'foobar: remove of file not supported'):
self.assertRaisesException(
wiredtiger.WiredTigerError,
lambda: fs.fs_remove(session, 'foobar', 0))
self.assertEquals(fs.fs_directory_list(session, '', ''),
['foobar'])
else:
fs.fs_remove(session, 'foobar', 0)
self.assertEquals(fs.fs_directory_list(session, '', ''), [])
fs.terminate(session)
ss.terminate(session)
def test_ss_write_read(self):
# Write and read to a file non-sequentially.
session = self.session
ss = self.get_storage_source()
# Since this class has multiple tests, append test name to the prefix to
# avoid namespace collison. 0th element on the stack is the current function.
prefix = self.bucket_prefix + inspect.stack()[0][3] + '/'
cachedir = self.bucket + '_cache'
os.mkdir(cachedir)
# Directory store needs the bucket created as a directory on the filesystem.
if self.ss_name == 'dir_store':
os.mkdir(self.bucket)
fs = ss.ss_customize_file_system(session, self.bucket, self.auth_token,
self.get_fs_config(prefix, cachedir))
# We call these 4K chunks of data "blocks" for this test, but that doesn't
# necessarily relate to WT block sizing.
nblocks = 1000
block_size = 4096
f = open('abc', 'wb')
# Create some blocks filled with 'a', etc.
a_block = ('a' * block_size).encode()
b_block = ('b' * block_size).encode()
c_block = ('c' * block_size).encode()
file_size = nblocks * block_size
# Write all blocks as 'a', but in reverse order.
for pos in range(file_size - block_size, 0, -block_size):
f.seek(pos)
f.write(a_block)
# Write the even blocks as 'b', forwards.
for pos in range(0, file_size, block_size * 2):
f.seek(pos)
f.write(b_block)
# Write every third block as 'c', backwards.
for pos in range(file_size - block_size, 0, -block_size * 3):
f.seek(pos)
f.write(c_block)
f.close()
# Flushing copies the file into the file system.
ss.ss_flush(session, fs, 'abc', 'abc', None)
ss.ss_flush_finish(session, fs, 'abc', 'abc', None)
# Use the file system to open and read the file.
# We do this twice, and between iterations, we remove the cached file to make sure
# it is copied back from the bucket.
#
# XXX: this uses knowledge of the implementation, but at the current time,
# we don't have a way via the API to "age out" a file from the cache.
for i in range(0, 2):
in_block = bytes(block_size)
fh = fs.fs_open_file(session, 'abc',
FileSystem.open_file_type_data,
FileSystem.open_readonly)
# Do some spot checks, reading non-sequentially.
fh.fh_read(session, 500 * block_size,
in_block) # divisible by 2, not 3
self.assertEquals(in_block, b_block)
fh.fh_read(session, 333 * block_size,
in_block) # divisible by 3, not 2
self.assertEquals(in_block, c_block)
fh.fh_read(session, 401 * block_size,
in_block) # not divisible by 2 or 3
self.assertEquals(in_block, a_block)
# Read the whole file, backwards checking to make sure
# each block was written correctly.
for block_num in range(nblocks - 1, 0, -1):
pos = block_num * block_size
fh.fh_read(session, pos, in_block)
if block_num % 3 == 0:
self.assertEquals(in_block, c_block)
elif block_num % 2 == 0:
self.assertEquals(in_block, b_block)
else:
self.assertEquals(in_block, a_block)
fh.close(session)
os.remove(os.path.join(cachedir, 'abc'))
ss.terminate(session)
def create_with_fs(self, fs, fname):
session = self.session
f = open(fname, 'wb')
f.write('some stuff'.encode())
f.close()
cachedir1 = "./cache1"
cachedir2 = "./cache2"
# Add a suffix to each in a list.
def suffix(self, lst, sfx):
return [x + '.' + sfx for x in lst]
def check_dirlist(self, fs, prefix, expect):
# We don't require any sorted output for directory lists,
# so we'll sort before comparing.'
got = sorted(fs.fs_directory_list(self.session, '', prefix))
expect = sorted(self.suffix(expect, 'wtobj'))
self.assertEquals(got, expect)
# Check for data files in the WiredTiger home directory.
def check_home(self, expect):
# Get list of all .wt files in home, prune out the WiredTiger produced ones.
got = sorted(list(os.listdir(self.home)))
got = [
x for x in got
if not x.startswith('WiredTiger') and x.endswith('.wt')
]
expect = sorted(self.suffix(expect, 'wt'))
self.assertEquals(got, expect)
# Check that objects are "in the cloud" for the directory store after a flush.
# Using the directory storage module, they are actually going to be in either
# bucket or bucket1.
def check_local_objects(self, expect1, expect2):
if self.ss_name != 'dir_store':
return
got = sorted(list(os.listdir(self.bucket)))
expect = sorted(self.suffix(expect1, 'wtobj'))
self.assertEquals(got, expect)
got = sorted(list(os.listdir(self.bucket1)))
expect = sorted(self.suffix(expect2, 'wtobj'))
self.assertEquals(got, expect)
# Check that objects are in the cache directory after flush_finish.
def check_caches(self, expect1, expect2):
got = sorted(list(os.listdir(self.cachedir1)))
expect = sorted(self.suffix(expect1, 'wtobj'))
self.assertEquals(got, expect)
got = sorted(list(os.listdir(self.cachedir2)))
expect = sorted(self.suffix(expect2, 'wtobj'))
self.assertEquals(got, expect)
def create_wt_file(self, name):
with open(name + '.wt', 'w') as f:
f.write('hello')
def test_ss_file_systems(self):
# Test using various buckets, hosts.
session = self.session
ss = self.get_storage_source()
# Since this class has multiple tests, append test name to the prefix to
# avoid namespace collison. 0th element on the stack is the current function.
prefix = self.bucket_prefix + inspect.stack()[0][3] + '/'
# Directory store needs the bucket created as a directory on the filesystem.
if self.ss_name == 'dir_store':
os.mkdir(self.bucket)
os.mkdir(self.bucket1)
os.mkdir(self.cachedir1)
os.mkdir(self.cachedir2)
bad_bucket = "./objects_BAD"
bad_cachedir = '/BAD'
# Create file system objects. First try some error cases.
errmsg = '/No such /'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: ss.ss_customize_file_system(
session, self.bucket, self.auth_token,
self.get_fs_config(prefix, bad_cachedir)), errmsg)
# S3 store expects a region with the bucket
if self.ss_name == 's3_store':
bad_bucket += ';us-east-2'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: ss.ss_customize_file_system(
session, bad_bucket, self.auth_token,
self.get_fs_config(prefix, self.cachedir1)), errmsg)
# For directory store - Create an empty file, try to use it as a directory.
if self.ss_name == 'dir_store':
with open("some_file", "w"):
pass
errmsg = '/Invalid argument/'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: ss.ss_customize_file_system(
session, "some_file", self.auth_token, ',cache_directory='
+ self.bucket), errmsg)
# Now create some file systems that should succeed.
# Use either different bucket directories or different prefixes,
# so activity that happens in the various file systems should be independent.
fs1 = ss.ss_customize_file_system(
session, self.bucket, self.auth_token,
self.get_fs_config(prefix, self.cachedir1))
fs2 = ss.ss_customize_file_system(
session, self.bucket1, self.auth_token,
self.get_fs_config(prefix, self.cachedir2))
# Create files in the wt home directory.
for a in ['beagle', 'bird', 'bison', 'bat']:
self.create_wt_file(a)
for a in ['cat', 'cougar', 'coyote', 'cub']:
self.create_wt_file(a)
# Everything is in wt home, nothing in the file system yet.
self.check_home([
'beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub'
])
self.check_dirlist(fs1, '', [])
self.check_dirlist(fs2, '', [])
self.check_caches([], [])
self.check_local_objects([], [])
# A flush copies to the cloud, nothing is removed.
ss.ss_flush(session, fs1, 'beagle.wt', 'beagle.wtobj')
self.check_home([
'beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub'
])
self.check_dirlist(fs1, '', ['beagle'])
self.check_dirlist(fs2, '', [])
self.check_caches([], [])
self.check_local_objects(['beagle'], [])
# Bad file to flush.
errmsg = '/No such file/'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: ss.ss_flush(session, fs1, 'bad.wt', 'bad.wtobj'), errmsg)
# It's okay to flush again, nothing changes.
ss.ss_flush(session, fs1, 'beagle.wt', 'beagle.wtobj')
self.check_home([
'beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub'
])
self.check_dirlist(fs1, '', ['beagle'])
self.check_dirlist(fs2, '', [])
self.check_caches([], [])
self.check_local_objects(['beagle'], [])
# When we flush_finish, the local file will be in both the local and cache directory.
ss.ss_flush_finish(session, fs1, 'beagle.wt', 'beagle.wtobj')
self.check_home([
'beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub'
])
self.check_dirlist(fs1, '', ['beagle'])
self.check_dirlist(fs2, '', [])
self.check_caches(['beagle'], [])
self.check_local_objects(['beagle'], [])
# Do a some more in each file system.
ss.ss_flush(session, fs1, 'bison.wt', 'bison.wtobj')
ss.ss_flush(session, fs2, 'cat.wt', 'cat.wtobj')
ss.ss_flush(session, fs1, 'bat.wt', 'bat.wtobj')
ss.ss_flush_finish(session, fs2, 'cat.wt', 'cat.wtobj')
ss.ss_flush(session, fs2, 'cub.wt', 'cub.wtobj')
ss.ss_flush_finish(session, fs1, 'bat.wt', 'bat.wtobj')
self.check_home([
'beagle', 'bird', 'bison', 'bat', 'cat', 'cougar', 'coyote', 'cub'
])
self.check_dirlist(fs1, '', ['beagle', 'bat', 'bison'])
self.check_dirlist(fs2, '', ['cat', 'cub'])
self.check_caches(['beagle', 'bat'], ['cat'])
self.check_local_objects(['beagle', 'bat', 'bison'], ['cat', 'cub'])
# Test directory listing prefixes.
self.check_dirlist(fs1, '', ['beagle', 'bat', 'bison'])
self.check_dirlist(fs1, 'ba', ['bat'])
self.check_dirlist(fs1, 'be', ['beagle'])
self.check_dirlist(fs1, 'x', [])
# Terminate just one of the custom file systems.
# We should be able to terminate file systems, but we should
# also be able to terminate the storage source without terminating
# all the file systems we created.
fs1.terminate(session)
ss.terminate(session)
class test_schema05(TieredConfigMixin, wttest.WiredTigerTestCase):
"""
Test indices with a custom extractor.
This test is the same as test_schema04, except that rows
are comma separated values (CSV) and we use a custom
extractor to pull out the index keys.
Our set of rows looks like a multiplication table:
row '0': '0,0,0,0,0,0'
row '1': '0,1,2,3,4,5'
row '2': '0,2,4,6,8,10'
with the twist that entries are mod 100. So, looking further:
row '31': '0,31,62,93,24,55'
Each column is placed into its own index. The mod twist,
as well as the 0th column, guarantees we'll have some duplicates.
"""
nentries = 1000
nindices = 6
index = [
('index-before', { 'create_index' : 0 }),
('index-during', { 'create_index' : 1 }),
('index-after', { 'create_index' : 2 }),
]
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, index)
def conn_extensions(self, extlist):
extlist.skip_if_missing = True
extlist.extension('extractors', 'csv')
return self.tiered_conn_extensions(extlist)
def create_indices(self):
# Create self.nindices index files, each with a column from the CSV
for i in range(0, self.nindices):
si = str(i)
self.session.create('index:schema05:x' + si,
'key_format=S,columns=(key),'
'extractor=csv,app_metadata={"format" : "S",' +
'"field" : "' + si + '"}')
def drop_indices(self):
for i in range(0, self.nindices):
self.dropUntilSuccess(self.session, "index:schema05:x" + str(i))
def csv(self, s, i):
return s.split(',')[i]
# We split the population into two phases
# (in anticipation of future tests that create
# indices between the two population steps).
def populate(self, phase):
cursor = self.session.open_cursor('table:schema05', None, None)
if phase == 0:
range_from = 0
range_to = self.nentries // 2
elif phase == 1:
range_from = self.nentries // 2
range_to = self.nentries - 5
else:
range_from = self.nentries - 5
range_to = self.nentries
for i in range(range_from, range_to):
# e.g. element 31 is '0,31,62,93,24,55'
cursor[i] = ','.join([str((i*j)%100) for j in
range(0, self.nindices)])
cursor.close()
def check_entries(self):
cursor = self.session.open_cursor('table:schema05', None, None)
icursor = []
for i in range(0, self.nindices):
icursor.append(self.session.open_cursor('index:schema05:x' + str(i),
None, None))
i = 0
for primkey, value in cursor:
# Check main table
expect = ','.join([str((i*j)%100) for j in range(0, self.nindices)])
self.assertEqual(i, primkey)
self.assertEqual(value, expect)
for idx in range(0, self.nindices):
c = icursor[idx]
indexkey = str((i*idx)%100)
c.set_key(indexkey)
self.assertEqual(c.search(), 0)
value = c.get_value()
while value != expect and \
self.csv(value, idx) == self.csv(expect, idx):
c.next()
value = c.get_value()
self.assertEqual(value, expect)
i += 1
cursor.close()
self.assertEqual(self.nentries, i)
for i in range(0, self.nindices):
icursor[i].close()
def test_index(self):
self.session.create("table:schema05", "key_format=i,value_format=S,"
"columns=(primarykey,value)")
if self.create_index == 0:
self.create_indices()
self.populate(0)
if self.create_index == 1:
self.create_indices()
self.populate(1)
if self.create_index == 2:
self.create_indices()
self.populate(2)
self.check_entries()
# Drop and recreate all indices, everything should be there.
self.drop_indices()
self.create_indices()
self.check_entries()
class test_alter01(TieredConfigMixin, wttest.WiredTigerTestCase):
name = "alter01"
entries = 100
# Data source types
types = [
('file', dict(uri='file:', use_cg=False, use_index=False)),
('lsm', dict(uri='lsm:', use_cg=False, use_index=False)),
('table-cg', dict(uri='table:', use_cg=True, use_index=False)),
('table-index', dict(uri='table:', use_cg=False, use_index=True)),
('table-simple', dict(uri='table:', use_cg=False, use_index=False)),
]
# Settings for access_pattern_hint
hints = [
('default', dict(acreate='')),
('none', dict(acreate='none')),
('random', dict(acreate='random')),
('sequential', dict(acreate='sequential')),
]
access_alter=('', 'none', 'random', 'sequential')
# Settings for cache_resident
resid = [
('default', dict(ccreate='')),
('false', dict(ccreate='false')),
('true', dict(ccreate='true')),
]
reopen = [
('no-reopen', dict(reopen=False)),
('reopen', dict(reopen=True)),
]
cache_alter=('', 'false', 'true')
# Build all scenarios
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, types, hints, resid, reopen)
def verify_metadata(self, metastr):
if metastr == '':
return
cursor = self.session.open_cursor('metadata:', None, None)
#
# Walk through all the metadata looking for the entries that are
# the file URIs for components of the table.
#
found = False
while True:
ret = cursor.next()
if ret != 0:
break
key = cursor.get_key()
check_meta = ((key.find("lsm:") != -1 or key.find("file:") != -1) \
and key.find(self.name) != -1)
if check_meta:
value = cursor[key]
found = True
self.assertTrue(value.find(metastr) != -1)
cursor.close()
self.assertTrue(found == True)
# Alter: Change the access pattern hint after creation
def test_alter01_access(self):
if self.is_tiered_scenario() and (self.uri == 'lsm:' or self.uri == 'file:'):
self.skipTest('Tiered storage does not support LSM or file URIs.')
uri = self.uri + self.name
create_params = 'key_format=i,value_format=i,'
complex_params = ''
#
# If we're not explicitly setting the parameter, then don't
# modify create_params to test using the default.
#
if self.acreate != '':
access_param = 'access_pattern_hint=%s' % self.acreate
create_params += '%s,' % access_param
complex_params += '%s,' % access_param
else:
# NOTE: This is hard-coding the default value. If the default
# changes then this will fail and need to be fixed.
access_param = 'access_pattern_hint=none'
if self.ccreate != '':
cache_param = 'cache_resident=%s' % self.ccreate
create_params += '%s,' % cache_param
complex_params += '%s,' % cache_param
else:
# NOTE: This is hard-coding the default value. If the default
# changes then this will fail and need to be fixed.
cache_param = 'cache_resident=false'
cgparam = ''
if self.use_cg or self.use_index:
cgparam = 'columns=(k,v),'
if self.use_cg:
cgparam += 'colgroups=(g0),'
self.session.create(uri, create_params + cgparam)
# Add in column group or index settings.
if self.use_cg:
cgparam = 'columns=(v),'
suburi = 'colgroup:' + self.name + ':g0'
self.session.create(suburi, complex_params + cgparam)
if self.use_index:
suburi = 'index:' + self.name + ':i0'
self.session.create(suburi, complex_params + cgparam)
# Put some data in table.
c = self.session.open_cursor(uri, None)
for k in range(self.entries):
c[k+1] = 1
c.close()
# Verify the string in the metadata
self.verify_metadata(access_param)
self.verify_metadata(cache_param)
# Run through all combinations of the alter commands
# for all allowed settings. This tests having only one or
# the other set as well as having both set. It will also
# cover trying to change the setting to its current value.
for a in self.access_alter:
alter_param = ''
access_str = ''
if a != '':
access_str = 'access_pattern_hint=%s' % a
for c in self.cache_alter:
alter_param = '%s' % access_str
cache_str = ''
if c != '':
cache_str = 'cache_resident=%s' % c
alter_param += ',%s' % cache_str
if alter_param != '':
self.session.alter(uri, alter_param)
if self.reopen:
self.reopen_conn()
special = self.use_cg or self.use_index
if not special:
self.verify_metadata(access_str)
self.verify_metadata(cache_str)
else:
self.session.alter(suburi, alter_param)
self.verify_metadata(access_str)
self.verify_metadata(cache_str)
class test_schema02(TieredConfigMixin, wttest.WiredTigerTestCase):
"""
Test basic operations
"""
nentries = 1000
types = [
('normal', dict(type='normal', idx_config='')),
('lsm', dict(type='lsm', idx_config=',type=lsm')),
]
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, types)
def expect_failure_colgroup(self, name, configstr, match):
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create("colgroup:" + name, configstr), match)
def test_colgroup_after_failure(self):
if self.is_tiered_scenario() and self.type == 'lsm':
self.skipTest('Tiered storage does not support LSM URIs.')
# bogus formats
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: self.session.create(
"table:main", "key_format=Z,value_format=S"),
"/Invalid type 'Z' found in format 'Z'/")
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: self.session.create(
"table:main", "key_format=S,value_format=Z"),
"/Invalid type 'Z' found in format 'Z'/")
# These should succeed
self.session.create(
"table:main", "key_format=iS,value_format=SiSi,"
"columns=(ikey,Skey,S1,i2,S3,i4),colgroups=(c1,c2)")
self.session.create("colgroup:main:c1", "columns=(S1,i2)")
def test_colgroup_failures(self):
if self.is_tiered_scenario() and self.type == 'lsm':
self.skipTest('Tiered storage does not support LSM URIs.')
# We skip testing the tiered storage scenarios as we fail to create
# column groups in tiered storage scenarios. We should fix this issue
# and then remove the condition to skip tests. FIXME: WT-9048
if self.is_tiered_scenario():
self.skipTest('Tiered storage does not work with column groups.')
# too many columns
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: self.session.create(
"table:main", "key_format=S,"
"value_format=,columns=(a,b)"),
"/Number of columns in '\(a,b\)' does not match "
"key format 'S' plus value format ''/")
# Note: too few columns is allowed
# expect this to work
self.session.create(
"table:main", "key_format=iS,value_format=SiSi,"
"columns=(ikey,Skey,S1,i2,S3,i4),"
"colgroups=(c1,c2)")
# bad table name
self.expect_failure_colgroup(
"nomatch:c", "columns=(S1,i2)",
"/Can't create 'colgroup:nomatch:c'"
" for non-existent table 'nomatch'/")
# colgroup not declared in initial create
self.expect_failure_colgroup(
"main:nomatch", "columns=(S1,i2)",
"/Column group 'nomatch' not found"
" in table 'main'/")
# bad column
self.expect_failure_colgroup("main:c1", "columns=(S1,i2,bad)",
"/Column 'bad' not found/")
# TODO: no columns allowed, or not?
#self.session.create("colgroup:main:c0", "columns=()")
# key in a column group
self.expect_failure_colgroup(
"main:c1", "columns=(ikey,S1,i2)",
"/A column group cannot store key column"
" 'ikey' in its value/")
# expect this to work
self.session.create("colgroup:main:c1", "columns=(S1,i2)")
# exclusive: no error message
self.expect_failure_colgroup("main:c1", "columns=(S1,i2),exclusive",
"")
# colgroup not declared in initial create
self.expect_failure_colgroup(
"main:c3", "columns=(S3,i4)", "/Column group 'c3' not found in"
" table 'main'/")
# this is the last column group, but there are missing columns
self.expect_failure_colgroup(
"main:c2", "columns=(S1,i4)",
"/Column 'S3' in 'table:main' does not"
" appear in a column group/")
# TODO: is repartitioning column groups allowed?
# this does not raise an error
# self.expect_failure_colgroup("main:c2", "columns=(S1,S3,i4)"
# expect this to work
self.session.create("colgroup:main:c2", "columns=(S3,i4)")
# expect these to work - each table name is a separate namespace
self.session.create(
"table:main2", "key_format=iS,value_format=SiSi,"
"columns=(ikey,Skey,S1,i2,S3,i4),colgroups=(c1,c2)")
self.session.create("colgroup:main2:c1", "columns=(S1,i2)")
self.session.create("colgroup:main2:c2", "columns=(S3,i4)")
def test_index(self):
if self.is_tiered_scenario() and self.type == 'lsm':
self.skipTest('Tiered storage does not support LSM URIs.')
self.session.create(
"table:main", "key_format=iS,value_format=SiSi,"
"columns=(ikey,Skey,S1,i2,S3,i4),colgroups=(c1,c2)")
# should be able to create colgroups before indices
self.session.create("colgroup:main:c2", "columns=(S3,i4)")
# should be able to create indices on all key combinations
self.session.create("index:main:ikey",
"columns=(ikey)" + self.idx_config)
self.session.create("index:main:Skey",
"columns=(Skey)" + self.idx_config)
self.session.create("index:main:ikeySkey",
"columns=(ikey,Skey)" + self.idx_config)
self.session.create("index:main:Skeyikey",
"columns=(Skey,ikey)" + self.idx_config)
# should be able to create indices on all value combinations
self.session.create("index:main:S1", "columns=(S1)" + self.idx_config)
self.session.create("index:main:i2", "columns=(i2)" + self.idx_config)
self.session.create("index:main:i2S1",
"columns=(i2,S1)" + self.idx_config)
self.session.create("index:main:S1i4",
"columns=(S1,i4)" + self.idx_config)
# somewhat nonsensical to repeat columns within an index, but allowed
self.session.create("index:main:i4S3i4S1",
"columns=(i4,S3,i4,S1)" + self.idx_config)
# should be able to create colgroups after indices
self.session.create("colgroup:main:c1", "columns=(S1,i2)")
self.populate()
# should be able to create indices after populating
self.session.create("index:main:i2S1i4",
"columns=(i2,S1,i4)" + self.idx_config)
self.check_entries()
self.check_indices()
def populate(self):
cursor = self.session.open_cursor('table:main', None, None)
for i in range(0, self.nentries):
square = i * i
cube = square * i
cursor[(i, 'key' + str(i))] = \
('val' + str(square), square, 'val' + str(cube), cube)
cursor.close()
def check_entries(self):
cursor = self.session.open_cursor('table:main', None, None)
# spot check via search
n = self.nentries
for i in (n // 5, 0, n - 1, n - 2, 1):
cursor.set_key(i, 'key' + str(i))
square = i * i
cube = square * i
cursor.search()
(s1, i2, s3, i4) = cursor.get_values()
self.assertEqual(s1, 'val' + str(square))
self.assertEqual(i2, square)
self.assertEqual(s3, 'val' + str(cube))
self.assertEqual(i4, cube)
i = 0
count = 0
# then check all via cursor
cursor.reset()
for ikey, skey, s1, i2, s3, i4 in cursor:
i = ikey
square = i * i
cube = square * i
self.assertEqual(ikey, i)
self.assertEqual(skey, 'key' + str(i))
self.assertEqual(s1, 'val' + str(square))
self.assertEqual(i2, square)
self.assertEqual(s3, 'val' + str(cube))
self.assertEqual(i4, cube)
count += 1
cursor.close()
self.assertEqual(count, n)
def check_indices(self):
# we check an index that was created before populating
cursor = self.session.open_cursor('index:main:S1i4', None, None)
count = 0
n = self.nentries
for s1key, i4key, s1, i2, s3, i4 in cursor:
i = int(i4key**(1 / 3.0) + 0.0001) # cuberoot
#self.tty('index:main:S1i4[' + str(i) + '] (' +
# str(s1key) + ',' +
# str(i4key) + ') -> (' +
# str(s1) + ',' +
# str(i2) + ',' +
# str(s3) + ',' +
# str(i4) + ')')
self.assertEqual(s1key, s1)
self.assertEqual(i4key, i4)
ikey = i
skey = 'key' + str(i)
square = i * i
cube = square * i
self.assertEqual(ikey, i)
self.assertEqual(skey, 'key' + str(i))
self.assertEqual(s1, 'val' + str(square))
self.assertEqual(i2, square)
self.assertEqual(s3, 'val' + str(cube))
self.assertEqual(i4, cube)
count += 1
cursor.close()
self.assertEqual(count, n)
# we check an index that was created after populating
cursor = self.session.open_cursor('index:main:i2S1i4', None, None)
count = 0
for i2key, s1key, i4key, s1, i2, s3, i4 in cursor:
i = int(i4key**(1 / 3.0) + 0.0001) # cuberoot
#self.tty('index:main:i2S1i4[' + str(i) + '] (' +
# str(i2key) + ',' +
# str(s1key) + ',' +
# str(i4key) + ') -> (' +
# str(s1) + ',' +
# str(i2) + ',' +
# str(s3) + ',' +
# str(i4) + ')')
self.assertEqual(i2key, i2)
self.assertEqual(s1key, s1)
self.assertEqual(i4key, i4)
ikey = i
skey = 'key' + str(i)
square = i * i
cube = square * i
self.assertEqual(ikey, i)
self.assertEqual(skey, 'key' + str(i))
self.assertEqual(s1, 'val' + str(square))
self.assertEqual(i2, square)
self.assertEqual(s3, 'val' + str(cube))
self.assertEqual(i4, cube)
count += 1
cursor.close()
self.assertEqual(count, n)
def test_colgroups(self):
if self.is_tiered_scenario() and self.type == 'lsm':
self.skipTest('Tiered storage does not support LSM URIs.')
self.session.create(
"table:main", "key_format=iS,value_format=SiSi,"
"columns=(ikey,Skey,S1,i2,S3,i4),colgroups=(c1,c2)")
self.session.create("colgroup:main:c1", "columns=(S1,i2)")
self.session.create("colgroup:main:c2", "columns=(S3,i4)")
self.populate()
self.check_entries()
class test_tiered14(wttest.WiredTigerTestCase, TieredConfigMixin):
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered14',
tiered_only=True)
uri = "table:test_tiered14-{}" # format for subtests
# FIXME-WT-7833: enable the commented scenarios and run the
# test with the --long option.
# The multiplier makes the size of keys and values progressively larger.
# A multiplier of 0 makes the keys and values a single length.
multiplier = [
('0', dict(multiplier=0)),
('S', dict(multiplier=1)),
('M', dict(multiplier=10)),
#('L', dict(multiplier=100, long_only=True)),
#('XL', dict(multiplier=1000, long_only=True)),
]
keyfmt = [
('integer', dict(keyfmt='i')),
('string', dict(keyfmt='S')),
]
dataset = [
('simple', dict(dataset='simple')),
#('complex', dict(dataset='complex', long_only=True)),
]
scenarios = make_scenarios(multiplier, keyfmt, dataset, storage_sources)
def conn_config(self):
return TieredConfigMixin.conn_config(self)
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
def progress(self, s):
outstr = "testnum {}, position {}: {}".format(self.testnum,
self.position, s)
self.verbose(3, outstr)
self.pr(outstr)
# Run a sequence of operations, indicated by a string.
# a = add some number of keys
# u = update some number of keys
# c = checkpoint
# r = reopen
# f = flush_tier
# . = check to make sure all expected values are present
#
# We require a unique test number so we get can generate a different uri from
# previous runs. A different approach is to drop the uri, but then we need to
# remove the bucket and cache, which is specific to the storage source extension.
def playback(self, testnum, ops):
self.testnum = testnum
self.position = -1
uri = self.uri.format(testnum)
self.progress('Running ops: {} using uri {}'.format(ops, uri))
if self.dataset == 'simple':
ds = TrackedSimpleDataSet(self,
uri,
self.multiplier,
key_format=self.keyfmt)
elif self.dataset == 'complex':
ds = TrackedComplexDataSet(self,
uri,
self.multiplier,
key_format=self.keyfmt)
# Populate for a tracked data set is needed to create the uri.
ds.populate()
inserted = 0
# At the end of the sequence of operations, do a final check ('.').
for op in ops + '.':
self.position += 1
try:
if op == 'f':
self.progress('flush_tier')
self.session.flush_tier(None)
elif op == 'c':
self.progress('checkpoint')
self.session.checkpoint()
elif op == 'r':
self.progress('reopen')
self.reopen_conn()
elif op == 'a':
self.progress('add')
n = random.randrange(1, 101) # 1 <= n <= 100
ds.store_range(inserted, n)
inserted += n
elif op == 'u':
self.progress('update')
# only update the elements if enough have already been added.
n = random.randrange(1, 101) # 1 <= n <= 100
if n < inserted:
pos = random.randrange(0, inserted - n)
ds.store_range(pos, n)
elif op == '.':
self.progress('check')
ds.check()
except Exception as e:
self.progress('Failed at position {} in {}: {}'.format(
idx, ops, str(e)))
raise (e)
# Test tiered storage with checkpoints and flush_tier calls.
def test_tiered(self):
random.seed(0)
# Get started with a fixed sequence of basic operations.
# There's no particular reason to start with this sequence.
testnum = 0
self.playback(testnum,
"aaaaacaaa.uucrauaf.aauaac.auu.aacrauafa.uruua.")
for i in range(0, 10):
testnum += 1
# Generate a sequence of operations that is heavy on additions and updates.
s = ''.join(random.choices('aaaaauuuuufcr.', k=self.num_ops))
self.playback(testnum, s)
for i in range(0, 10):
testnum += 1
# Generate a sequence of operations that is has a greater mix of 'operational' functions.
s = ''.join(random.choices('aufcr.', k=self.num_ops))
self.playback(testnum, s)
class test_tiered07(wttest.WiredTigerTestCase, TieredConfigMixin):
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered07',
tiered_only=True)
# FIXME-WT-8897 Disabled S3 (only indexing dirstore in storage sources) as S3 directory listing
# is interpreting a directory to end in a '/', whereas the code in the tiered storage doesn't
# expect that. Enable when fixed.
# Make scenarios for different cloud service providers
flush_obj = [
('ckpt', dict(first_ckpt=True)),
('no_ckpt', dict(first_ckpt=False)),
]
tiered_storage_dirstore_source = storage_sources[:1]
scenarios = make_scenarios(flush_obj, tiered_storage_dirstore_source)
uri = "table:abc"
uri2 = "table:ab"
uri3 = "table:abcd"
uri4 = "table:abcde"
localuri = "table:local"
newuri = "table:tier_new"
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
def conn_config(self):
return TieredConfigMixin.conn_config(self)
def check(self, tc, base, n):
get_check(self, tc, 0, n)
# Test calling schema APIs with a tiered table.
def test_tiered(self):
# Create a new tiered table.
self.pr('create table')
self.session.create(self.uri, 'key_format=S,value_format=S')
self.pr('create table 2')
self.session.create(self.uri2, 'key_format=S,value_format=S')
self.pr('create table 3')
self.session.create(self.uri3, 'key_format=S,value_format=S')
self.pr('create table local')
self.session.create(
self.localuri,
'key_format=S,value_format=S,tiered_storage=(name=none)')
# Rename is not supported for tiered tables.
msg = "/is not supported/"
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError, lambda: self.assertEquals(
self.session.rename(self.uri, self.newuri, None), 0), msg)
# Add some data and flush tier.
self.pr('add one item to all tables')
c = self.session.open_cursor(self.uri)
c["0"] = "0"
self.check(c, 0, 1)
c.close()
c = self.session.open_cursor(self.uri2)
c["0"] = "0"
self.check(c, 0, 1)
c.close()
c = self.session.open_cursor(self.uri3)
c["0"] = "0"
self.check(c, 0, 1)
c.close()
c = self.session.open_cursor(self.localuri)
c["0"] = "0"
c.close()
if (self.first_ckpt):
self.session.checkpoint()
self.pr('After data, call flush_tier')
self.session.flush_tier(None)
if (not self.first_ckpt):
self.session.checkpoint()
# Drop table.
self.pr('call drop')
self.session.drop(self.localuri)
self.session.drop(self.uri)
# By default, the remove_files configuration for drop is true. This means that the
# drop operation for tiered tables should both remove the files from the metadata
# file and remove the corresponding local object files in the directory.
self.assertFalse(os.path.isfile("abc-0000000001.wtobj"))
self.assertFalse(os.path.isfile("abc-0000000002.wtobj"))
# Dropping a table using the force setting should succeed even if the table does not exist.
self.pr('drop with force')
self.session.drop(self.localuri, 'force=true')
self.session.drop(self.uri, 'force=true')
# Dropping a table should not succeed if the table does not exist.
# Test dropping a table that was previously dropped.
self.assertRaises(wiredtiger.WiredTigerError,
lambda: self.session.drop(self.localuri, None))
# Test dropping a table that does not exist.
self.assertRaises(
wiredtiger.WiredTigerError,
lambda: self.session.drop("table:random_non_existent", None))
# Create new table with same name. This should error.
self.session.create(self.newuri, 'key_format=S')
# If we didn't do a checkpoint before the flush_tier then creating with the same name
# will succeed because no bucket objects were created.
if (self.first_ckpt):
self.pr('check cannot create with same name')
self.assertRaises(
wiredtiger.WiredTigerError,
lambda: self.session.create(self.uri, 'key_format=S'))
else:
self.session.create(self.uri, 'key_format=S')
# Make sure there was no problem with overlapping table names.
self.pr('check original similarly named tables')
c = self.session.open_cursor(self.uri2)
self.check(c, 0, 1)
c.close()
c = self.session.open_cursor(self.uri3)
self.check(c, 0, 1)
c.close()
# Create new table with new name.
self.pr('create new table')
self.session.create(self.newuri, 'key_format=S')
# Test the drop operation without removing associated files.
self.session.create(self.uri4, 'key_format=S,value_format=S')
self.session.drop(self.uri4, 'remove_files=false')
self.assertTrue(os.path.isfile("abcde-0000000001.wtobj"))
class test_alter04(TieredConfigMixin, wttest.WiredTigerTestCase):
name = "alter04"
entries = 100
cache_alter = ('1M', '100K')
# Settings for os_cache[_dirty]_max.
types = [
('file', dict(uri='file:', use_cg=False, use_index=False)),
('lsm', dict(uri='lsm:', use_cg=False, use_index=False)),
('table-cg', dict(uri='table:', use_cg=True, use_index=False)),
('table-index', dict(uri='table:', use_cg=False, use_index=True)),
('table-simple', dict(uri='table:', use_cg=False, use_index=False)),
]
sizes = [
('default', dict(ocreate='')),
('1M', dict(ocreate='1M')),
('200K', dict(ocreate='200K')),
]
reopen = [
('no-reopen', dict(reopen=False)),
('reopen', dict(reopen=True)),
]
settings = [
('cache', dict(setting='os_cache_max')),
('cache_dirty', dict(setting='os_cache_dirty_max')),
]
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, types, sizes, reopen,
settings)
def verify_metadata(self, metastr):
if metastr == '':
return
cursor = self.session.open_cursor('metadata:', None, None)
#
# Walk through all the metadata looking for the entries that are
# the file URIs for components of the table.
#
found = False
while True:
ret = cursor.next()
if ret != 0:
break
key = cursor.get_key()
check_meta = ((key.find("lsm:") != -1 or key.find("file:") != -1) \
and key.find(self.name) != -1)
if check_meta:
value = cursor[key]
found = True
self.assertTrue(value.find(metastr) != -1)
cursor.close()
self.assertTrue(found == True)
# Alter: Change the setting after creation
def test_alter04_cache(self):
if self.is_tiered_scenario() and (self.uri == 'lsm:'
or self.uri == 'file:'):
self.skipTest('Tiered storage does not support LSM or file URIs.')
uri = self.uri + self.name
create_params = 'key_format=i,value_format=i,'
complex_params = ''
#
# If we're not explicitly setting the parameter, then don't
# modify create_params to test using the default.
#
if self.ocreate != '':
new_param = '%s=%s' % (self.setting, self.ocreate)
create_params += '%s,' % new_param
complex_params += '%s,' % new_param
else:
# NOTE: This is hard-coding the default value. If the default
# changes then this will fail and need to be fixed.
new_param = '%s=0' % self.setting
cgparam = ''
if self.use_cg or self.use_index:
cgparam = 'columns=(k,v),'
if self.use_cg:
cgparam += 'colgroups=(g0),'
self.session.create(uri, create_params + cgparam)
# Add in column group or index settings.
if self.use_cg:
cgparam = 'columns=(v),'
suburi = 'colgroup:' + self.name + ':g0'
self.session.create(suburi, complex_params + cgparam)
if self.use_index:
suburi = 'index:' + self.name + ':i0'
self.session.create(suburi, complex_params + cgparam)
# Put some data in table.
c = self.session.open_cursor(uri, None)
for k in range(self.entries):
c[k + 1] = 1
c.close()
# Verify the string in the metadata
self.verify_metadata(new_param)
# Run through all combinations of the alter commands
# for all allowed settings.
for a in self.cache_alter:
alter_param = '%s=%s' % (self.setting, a)
self.session.alter(uri, alter_param)
if self.reopen:
self.reopen_conn()
special = self.use_cg or self.use_index
if not special:
self.verify_metadata(alter_param)
else:
self.session.alter(suburi, alter_param)
self.verify_metadata(alter_param)
class test_export01(TieredConfigMixin, wttest.WiredTigerTestCase):
dir = 'backup.dir'
types = [
('table', dict(type='table:')),
]
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, types)
def test_export(self):
uri_a = self.type + "exporta"
uri_b = self.type + "exportb"
uri_c = self.type + "exportc"
# Create a few tables.
self.session.create(uri_a)
self.session.create(uri_b)
self.session.create(uri_c)
# Insert some records.
c1 = self.session.open_cursor(uri_a)
c1["k1"] = "v1"
c1.close()
c2 = self.session.open_cursor(uri_b)
c2["k2"] = "v2"
c2.close()
c3 = self.session.open_cursor(uri_c)
c3["k3"] = "v3"
c3.close()
self.session.checkpoint()
if self.is_tiered_scenario():
self.session.flush_tier(None)
# Open a special backup cursor for export operation.
export_cursor = self.session.open_cursor('backup:export', None, None)
os.mkdir(self.dir)
copy_wiredtiger_home(self, '.', self.dir)
self.assertTrue(os.path.isfile("WiredTiger.export"))
export_cursor.close()
# The export file should be removed from the home directory.
self.assertFalse(os.path.isfile("WiredTiger.export"))
# The export file should exist in the backup directory.
self.assertTrue(
os.path.isfile(os.path.join(self.dir, "WiredTiger.export")))
def test_export_restart(self):
uri_a = self.type + "exporta"
uri_b = self.type + "exportb"
uri_c = self.type + "exportc"
# Create two tables.
self.session.create(uri_a)
self.session.create(uri_b)
# Insert some records.
c4 = self.session.open_cursor(uri_a)
c4["k4"] = "v4"
c4.close()
c5 = self.session.open_cursor(uri_b)
c5["k5"] = "v5"
c5.close()
self.session.checkpoint()
if self.is_tiered_scenario():
self.session.flush_tier(None)
# Open a special backup cursor for export operation.
main_cursor = self.session.open_cursor('backup:export', None, None)
# Copy the file so that we have more information if WT-9203 ever happens again.
shutil.copyfile('WiredTiger.export', 'WiredTiger.export.original')
# Copy the main database to another directory, including the WiredTiger.export file.
os.mkdir(self.dir)
copy_wiredtiger_home(self, '.', self.dir)
main_cursor.close()
self.close_conn()
# Open a connection and session on the directory copy.
self.conn = self.setUpConnectionOpen(self.dir)
self.session = self.setUpSessionOpen(self.conn)
# Create a third table and drop the second table.
self.session.create(uri_c)
c6 = self.session.open_cursor(uri_c)
c6["k6"] = "k6"
c6.close()
self.session.checkpoint()
if self.is_tiered_scenario():
self.session.flush_tier(None)
self.session.drop(uri_b)
# Open an export cursor on the database copy.
wt_export_path = os.path.join(self.dir, "WiredTiger.export")
export_cursor = self.session.open_cursor('backup:export', None, None)
# Copy the file so that we have more information if WT-9203 ever happens again.
shutil.copyfile(wt_export_path,
os.path.join(self.dir, "WiredTiger.export.backup"))
self.assertTrue(os.path.isfile(wt_export_path))
# The information for the third table should exist in the WiredTiger.export file
# but the information for the second table should not exist in the file.
with open(wt_export_path, "r") as export_file:
export_file_string = export_file.read()
self.assertFalse("exportb" in export_file_string)
self.assertTrue("exportc" in export_file_string)
export_cursor.close()
class test_schema08(TieredConfigMixin, wttest.WiredTigerTestCase,
suite_subprocess):
# We want to copy, truncate and run recovery so keep the log
# file small and don't pre-allocate any. We expect a small log.
conn_config_string = 'log=(enabled,file_max=100k,prealloc=false,remove=false),'
types = [
('file', dict(uri='file:', use_cg=False, use_index=False)),
('lsm', dict(uri='lsm:', use_cg=False, use_index=False)),
('table-cg', dict(uri='table:', use_cg=True, use_index=False)),
('table-index', dict(uri='table:', use_cg=False, use_index=True)),
('table-simple', dict(uri='table:', use_cg=False, use_index=False)),
]
ops = [
('none', dict(schema_ops='none')),
('alter', dict(schema_ops='alter')),
('drop', dict(schema_ops='drop')),
('rename', dict(schema_ops='rename')),
]
ckpt = [
('no_ckpt', dict(ckpt=False)),
('with_ckpt', dict(ckpt=True)),
]
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources, types, ops, ckpt)
count = 0
lsns = []
backup_pfx = "BACKUP."
# Setup connection config.
def conn_config(self):
return self.conn_config_string + self.tiered_conn_config()
def do_alter(self, uri, suburi):
alter_param = 'cache_resident=true'
self.session.alter(uri, alter_param)
if suburi != None:
self.session.alter(suburi, alter_param)
def do_ops(self, uri, suburi):
if (self.schema_ops == 'none'):
return
if (self.schema_ops == 'alter'):
self.do_alter(uri, suburi)
elif (self.schema_ops == 'drop'):
self.session.drop(uri, None)
elif (self.schema_ops == 'rename'):
newuri = self.uri + "new-table"
self.session.rename(uri, newuri, None)
# Count actual log records in the log. Log cursors walk the individual
# operations of a transaction as well as the entire record. Skip counting
# any individual commit operations and only count entire records.
def find_logrecs(self):
self.count = 0
self.session.log_flush('sync=on')
c = self.session.open_cursor('log:', None, None)
self.lsns.append(0)
while c.next() == 0:
# lsn.file, lsn.offset, opcount
keys = c.get_key()
# We don't expect to need more than one log file. We only store
# the offsets in a list so assert lsn.file is 1.
self.assertTrue(keys[0] == 1)
# Only count whole records, which is when opcount is zero.
# If opcount is not zero it is an operation of a commit.
# Skip LSN 128, that is a system record and its existence
# is assumed within the system.
if keys[2] == 0 and keys[1] != 128:
self.count += 1
self.lsns.append(keys[1])
c.close()
self.pr("Find " + str(self.count) + " logrecs LSNS: ")
self.pr(str(self.lsns))
def make_backups(self):
# With the connection still open, copy files to the new directory.
# Make an initial copy as well as a copy for each LSN we save.
# Truncate the log to the appropriate offset as we make each copy.
olddir = "."
log1 = 'WiredTigerLog.0000000001'
for lsn in self.lsns:
newdir = self.backup_pfx + str(lsn)
shutil.rmtree(newdir, ignore_errors=True)
os.mkdir(newdir)
for fname in os.listdir(olddir):
fullname = os.path.join(olddir, fname)
# Skip lock file on Windows since it is locked
if os.path.isfile(fullname) and \
"WiredTiger.lock" not in fullname and \
"Tmplog" not in fullname and \
"Preplog" not in fullname:
shutil.copy(fullname, newdir)
# Truncate the file to the LSN offset.
# NOTE: This removes the record at that offset
# resulting in recovery running to just before
# that record.
if lsn != 0:
logf = os.path.join(newdir + '/' + log1)
f = open(logf, "r+")
f.truncate(lsn)
f.close()
# print "New size " + logf + ": " + str(os.path.getsize(logf))
def run_recovery(self, uri, suburi):
# With the connection still open, copy files to the new directory.
# Make an initial copy as well as a copy for each LSN we save.
# Truncate the log to the appropriate offset as we make each copy.
olddir = "."
errfile = "errfile.txt"
for lsn in self.lsns:
newdir = self.backup_pfx + str(lsn)
outfile = newdir + '.txt'
self.runWt(['-R', '-h', newdir, 'list', '-v'],
errfilename=errfile,
outfilename=outfile)
if os.path.isfile(errfile) and os.path.getsize(errfile) > 0:
self.check_file_contains(errfile, 'No such file or directory')
# Test that creating and dropping tables does not write individual
# log records.
def test_schema08_create(self):
if self.is_tiered_scenario() and (self.uri == 'lsm:'
or self.uri == 'file:'
or self.schema_ops == 'rename'):
self.skipTest(
'Tiered storage does not support LSM or file URIs, and also does not support the rename operation.'
)
self.count = 0
self.lsns = []
uri = self.uri + 'table0'
create_params = 'key_format=i,value_format=S,'
cgparam = ''
suburi = None
if self.use_cg or self.use_index:
cgparam = 'columns=(k,v),'
if self.use_cg:
cgparam += 'colgroups=(g0),'
# Create main table.
self.session.create(uri, create_params + cgparam)
# Checkpoint after the main table creation if wanted.
if self.ckpt:
self.session.checkpoint()
# Add in column group or index tables.
if self.use_cg:
# Create.
cgparam = 'columns=(v),'
suburi = 'colgroup:table0:g0'
self.session.create(suburi, cgparam)
if self.use_index:
# Create.
suburi = 'index:table0:i0'
self.session.create(suburi, cgparam)
self.do_ops(uri, suburi)
self.find_logrecs()
# print "Found " + str(self.count) + " log records"
if not self.is_tiered_scenario():
self.make_backups()
self.run_recovery(uri, suburi)
class test_tiered17(TieredConfigMixin, wttest.WiredTigerTestCase):
tiered_storage_sources = gen_tiered_storage_sources()
saved_conn = ''
uri = "table:test_tiered"
shutdown = [
('clean', dict(clean=True)),
('unclean', dict(clean=False)),
]
def conn_config(self):
if self.is_tiered_scenario():
self.saved_conn = get_conn_config(self) + ')'
return self.saved_conn
scenarios = make_scenarios(tiered_storage_sources, shutdown)
def get_object_files(self):
object_files = fnmatch.filter(os.listdir('.'),
"*.wtobj") + fnmatch.filter(
os.listdir('.'), '*.wt')
return object_files
def verify_checkpoint(self):
obj_files_orig = self.get_object_files()
ckpt_cursor = self.session.open_cursor(
self.uri, None, 'checkpoint=WiredTigerCheckpoint')
ckpt_cursor.close()
obj_files = self.get_object_files()
# Check that no additional object files have been created after opening the checkpoint.
self.assertTrue(len(obj_files_orig) == len(obj_files))
def populate(self):
# Create and populate a table.
self.session.create(self.uri, "key_format=S,value_format=S")
c = self.session.open_cursor(self.uri)
c["a"] = "a"
c["b"] = "b"
# Do a checkpoint and flush operation.
self.session.checkpoint()
self.session.flush_tier(None)
# Add more data but don't do a checkpoint or flush in the unclean shutdown scenario.
if not self.clean:
c["c"] = "c"
c["d"] = "d"
c.close()
def test_open_readonly_conn(self):
self.populate()
self.verify_checkpoint()
obj_files_orig = self.get_object_files()
# Re-open the connection but in readonly mode.
conn_params = 'readonly=true,' + self.saved_conn
self.reopen_conn(config=conn_params)
obj_files = self.get_object_files()
# Check that no additional object files have been created after re-opening the connection.
self.assertTrue(len(obj_files_orig) == len(obj_files))
self.close_conn()
# Check that no additional object files have been created after closing the connection.
obj_files = self.get_object_files()
self.assertTrue(len(obj_files_orig) == len(obj_files))
def test_open_readonly_cursor(self):
self.populate()
obj_files_orig = self.get_object_files()
# Open the database in readonly mode.
self.reopen_conn(config=self.saved_conn)
c = self.session.open_cursor(self.uri, None, "readonly=true")
obj_files = self.get_object_files()
# Check that no additional object files have been created after re-opening the connection.
self.assertTrue(len(obj_files_orig) == len(obj_files))
c.close()
self.close_conn()
# Check that no additional object files have been created after closing the connection.
obj_files = self.get_object_files()
self.assertTrue(len(obj_files_orig) == len(obj_files))
class test_schema03(TieredConfigMixin, wttest.WiredTigerTestCase):
"""
Test schemas - a 'predictably random' assortment of columns,
column groups and indices are created within tables, and are
created in various orders as much as the API allows. On some runs
the connection will be closed and reopened at a particular point
to test that the schemas (and data) are saved and read correctly.
The test is run multiple times, using scenarios.
The test always follows these steps:
- table: create tables
- colgroup0: create (some) colgroups
- index0: create (some) indices
- colgroup1: create (more) colgroups
- index1: create (more) indices
- populate0: populate 1st time
- index2: create (more) indices
- populate1: populate 2nd time (more key/values)
- check: check key/values
The variations represented by scenarios are:
- how many tables to create
- how many colgroups to create at each step (may be 0)
- how many indices to create at each step (may be 0)
- between each step, whether to close/reopen the connection
"""
# Boost cache size and number of sessions for this test
conn_config_string = 'cache_size=100m,session_max=1000,'
################################################################
# These three variables can be altered to help generate
# and pare down failing test cases.
# Set to true to get python test program fragment on stdout,
# used by show_python() below.
SHOW_PYTHON = False
# When SHOW_PYTHON is set, we print an enormous amount of output.
# To only print for a given scenario, set this
SHOW_PYTHON_ONLY_SCEN = None # could be e.g. [2] or [0,1]
# To print verbosely for only a given table, set this
SHOW_PYTHON_ONLY_TABLE = None # could be e.g. [2] or [0,1]
################################################################
# Set whenever we are working with a table
current_table = None
nentries = 50
# We need to have a large number of open files available
# to run this test. We probably don't need quite this many,
# but boost it up to this limit anyway.
OPEN_FILE_LIMIT = 1000
restart_scenarios = [
('table', dict(s_restart=['table'], P=0.3)),
('colgroup0', dict(s_restart=['colgroup0'], P=0.3)),
('index0', dict(s_restart=['index0'], P=0.3)),
('colgroup1', dict(s_restart=['colgroup1'], P=0.3)),
('index1', dict(s_restart=['index1'], P=0.3)),
('populate0', dict(s_restart=['populate0'], P=0.3)),
('index2', dict(s_restart=['index2'], P=0.3)),
('populate1', dict(s_restart=['populate1'], P=0.3)),
('ipop', dict(s_restart=['index0', 'populate0'], P=0.3)),
('all',
dict(s_restart=[
'table', 'colgroup0', 'index0', 'colgroup1', 'index1',
'populate0', 'index2', 'populate1'
],
P=1.0)),
]
ntable_scenarios = wtscenario.quick_scenarios('s_ntable', [1, 2, 5, 8],
[1.0, 0.4, 0.5, 0.5])
ncolgroup_scenarios = wtscenario.quick_scenarios(
's_colgroup', [[1, 0], [0, 1], [2, 4], [8, 5]], [1.0, 0.2, 0.3, 1.0])
nindex_scenarios = wtscenario.quick_scenarios(
's_index', [[1, 1, 1], [3, 2, 1], [5, 1, 3]], [1.0, 0.5, 1.0])
idx_args_scenarios = wtscenario.quick_scenarios(
's_index_args', ['', ',type=file', ',type=lsm'], [0.5, 0.3, 0.2])
table_args_scenarios = wtscenario.quick_scenarios(
's_extra_table_args', ['', ',type=file', ',type=lsm'], [0.5, 0.3, 0.2])
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = wtscenario.make_scenarios(tiered_storage_sources,
restart_scenarios,
ntable_scenarios,
ncolgroup_scenarios,
nindex_scenarios,
idx_args_scenarios,
table_args_scenarios,
prune=30)
# Note: the set can be reduced here for debugging, e.g.
# scenarios = scenarios[40:44]
# or
# scenarios = [ scenarios[0], scenarios[30], scenarios[40] ]
#wttest.WiredTigerTestCase.printVerbose(2, 'test_schema03: running ' + \
# str(len(scenarios)) + ' of ' + \
# str(len(all_scenarios)) + ' possible scenarios')
# This test requires a large number of open files.
# Increase our resource limits before we start
def setUp(self):
if os.name == "nt":
self.skipTest('Unix specific test skipped on Windows')
self.origFileLimit = resource.getrlimit(resource.RLIMIT_NOFILE)
newlimit = (self.OPEN_FILE_LIMIT, self.origFileLimit[1])
if newlimit[0] > newlimit[1]:
self.skipTest('Require %d open files, only %d available' %
newlimit)
resource.setrlimit(resource.RLIMIT_NOFILE, newlimit)
super(test_schema03, self).setUp()
# Set up connection config.
def conn_config(self):
return self.conn_config_string + self.tiered_conn_config()
def tearDown(self):
super(test_schema03, self).tearDown()
resource.setrlimit(resource.RLIMIT_NOFILE, self.origFileLimit)
def gen_formats(self, rand, n, iskey):
result = ''
for i in range(0, n):
if rand.rand_range(0, 2) == 0:
result += 'S'
else:
result += 'i'
return result
def show_python(self, s):
if self.SHOW_PYTHON:
if self.SHOW_PYTHON_ONLY_TABLE == None or self.current_table in self.SHOW_PYTHON_ONLY_TABLE:
if self.SHOW_PYTHON_ONLY_SCEN == None or self.scenario_number in self.SHOW_PYTHON_ONLY_SCEN:
print(' ' + s)
def join_names(self, sep, prefix, list):
return sep.join([prefix + str(val) for val in list])
def create(self, what, tablename, whatname, columnlist, extra_args=''):
createarg = what + ":" + tablename + ":" + whatname
colarg = self.join_names(',', 'c', columnlist)
self.show_python("self.session.create('" + createarg +
"', 'columns=(" + colarg + ")" + extra_args + "')")
result = self.session.create(createarg,
"columns=(" + colarg + ")" + extra_args)
self.assertEqual(result, 0)
def finished_step(self, name):
if self.s_restart == name:
print(" # Reopening connection at step: " + name)
self.reopen_conn()
def test_schema(self):
if self.is_tiered_scenario() and (
self.s_index_args == ',type=lsm' or self.s_index_args
== ',type=file' or self.s_extra_table_args == ',type=lsm'
or self.s_extra_table_args == ',type=file'):
self.skipTest('Tiered storage does not support LSM or file URIs.')
rand = suite_random.suite_random()
if self.SHOW_PYTHON:
print(' ################################################')
print(' # Running scenario ' + str(self.scenario_number))
ntables = self.s_ntable
# Report known limitations in the test,
# we'll work around these later, in a loop where we don't want to print.
self.KNOWN_LIMITATION(
'Column groups created after indices confuses things')
# Column groups are created in two different times.
# We call these two batches 'createsets'.
# So we don't have the exactly the same number of column groups
# for each table, for tests that indicate >1 colgroup, we
# increase the number of column groups for each table
tabconfigs = []
for i in range(0, ntables):
self.current_table = i
tc = tabconfig()
tc.tablename = 't' + str(i)
tc.tableidx = i
tabconfigs.append(tc)
for createset in range(0, 2):
ncg = self.s_colgroup[createset]
if ncg > 1:
ncg += i
for k in range(0, ncg):
thiscg = cgconfig()
thiscg.createset = createset
# KNOWN LIMITATION: Column groups created after
# indices confuses things. So for now, put all
# column group creation in the first set.
# Remove this statement when the limitation is fixed.
thiscg.createset = 0
# END KNOWN LIMITATION
thiscg.cgname = 'g' + str(len(tc.cglist))
tc.cglist.append(thiscg)
# The same idea for indices, except that we create them in
# three sets
for createset in range(0, 3):
nindex = self.s_index[createset]
if nindex > 1:
nindex += i
for k in range(0, nindex):
thisidx = idxconfig()
thisidx.createset = createset
thisidx.idxname = 'i' + str(len(tc.idxlist))
thisidx.tab = tc
tc.idxlist.append(thisidx)
# We'll base the number of key/value columns
# loosely on the number of column groups and indices.
colgroups = len(tc.cglist)
indices = len(tc.idxlist)
nall = colgroups * 2 + indices
k = rand.rand_range(1, nall)
v = rand.rand_range(0, nall)
# we need at least one value per column group
if v < colgroups:
v = colgroups
tc.nkeys = k
tc.nvalues = v
tc.keyformats = self.gen_formats(rand, tc.nkeys, True)
tc.valueformats = self.gen_formats(rand, tc.nvalues, False)
# Simple naming (we'll test odd naming elsewhere):
# tables named 't0' --> 't<N>'
# within each table:
# columns named 'c0' --> 'c<N>'
# colgroups named 'g0' --> 'g<N>'
# indices named 'i0' --> 'i<N>'
config = ""
config += "key_format=" + tc.keyformats
config += ",value_format=" + tc.valueformats
config += ",columns=("
for j in range(0, tc.nkeys + tc.nvalues):
if j != 0:
config += ","
config += "c" + str(j)
config += "),colgroups=("
for j in range(0, len(tc.cglist)):
if j != 0:
config += ","
config += "g" + str(j)
config += ")"
config += self.s_extra_table_args
# indices are not declared here
self.show_python("self.session.create('table:" + tc.tablename +
"', '" + config + "')")
self.session.create("table:" + tc.tablename, config)
tc.columns_for_groups(list(range(tc.nkeys, tc.nkeys + tc.nvalues)))
tc.columns_for_indices(list(range(0, tc.nkeys + tc.nvalues)))
self.finished_step('table')
for createset in (0, 1):
# Create column groups in this set
# e.g. self.session.create("colgroup:t0:g1", "columns=(c3,c4)")
for tc in tabconfigs:
self.current_table = tc.tableidx
for cg in tc.cglist:
if cg.createset == createset:
self.create('colgroup', tc.tablename, cg.cgname,
cg.columns)
self.finished_step('colgroup' + str(createset))
# Create indices in this set
# e.g. self.session.create("index:t0:i1", "columns=(c3,c4)")
for tc in tabconfigs:
self.current_table = tc.tableidx
for idx in tc.idxlist:
if idx.createset == createset:
self.create('index', tc.tablename, idx.idxname,
idx.columns, self.s_index_args)
self.finished_step('index' + str(createset))
# populate first batch
for tc in tabconfigs:
self.current_table = tc.tableidx
max = rand.rand_range(0, self.nentries)
self.populate(tc, list(range(0, max)))
self.finished_step('populate0')
# Create indices in third set
for tc in tabconfigs:
for idx in tc.idxlist:
if idx.createset == 2:
self.create('index', tc.tablename, idx.idxname,
idx.columns)
self.finished_step('index2')
# populate second batch
for tc in tabconfigs:
self.current_table = tc.tableidx
self.populate(tc, list(range(tc.nentries, self.nentries)))
self.finished_step('populate1')
for tc in tabconfigs:
self.current_table = tc.tableidx
self.check_entries(tc)
def populate(self, tc, insertrange):
self.show_python("cursor = self.session.open_cursor('table:" +
tc.tablename + "', None, None)")
cursor = self.session.open_cursor('table:' + tc.tablename, None, None)
for i in insertrange:
key = tc.gen_keys(i)
val = tc.gen_values(i)
self.show_python("cursor.set_key(*" + str(key) + ")")
cursor.set_key(*key)
self.show_python("cursor.set_value(*" + str(val) + ")")
cursor.set_value(*val)
self.show_python("cursor.insert()")
cursor.insert()
tc.nentries += 1
self.show_python("cursor.close()")
cursor.close()
def check_one(self, name, cursor, key, val):
keystr = str(key)
valstr = str(val)
self.show_python('# search[' + name + '](' + keystr + ')')
self.show_python("cursor.set_key(*" + keystr + ")")
cursor.set_key(*key)
self.show_python("ok = cursor.search()")
ok = cursor.search()
self.show_python("self.assertEqual(ok, 0)")
self.assertEqual(ok, 0)
self.show_python("self.assertEqual(" + keystr + ", cursor.get_keys())")
self.assertEqual(key, cursor.get_keys())
self.show_python("self.assertEqual(" + valstr +
", cursor.get_values())")
self.assertEqual(val, cursor.get_values())
def check_entries(self, tc):
"""
Verify entries in the primary and index table
related to the tabconfig.
"""
self.show_python('# check_entries: ' + tc.tablename)
self.show_python("cursor = self.session.open_cursor('table:" +
tc.tablename + "', None, None)")
cursor = self.session.open_cursor('table:' + tc.tablename, None, None)
count = 0
for x in cursor:
count += 1
self.assertEqual(count, tc.nentries)
for i in range(0, tc.nentries):
key = tc.gen_keys(i)
val = tc.gen_values(i)
self.check_one(tc.tablename, cursor, key, val)
cursor.close()
self.show_python("cursor.close()")
# for each index, check each entry
for idx in tc.idxlist:
# Although it's possible to open an index on some partial
# list of columns, we'll keep it simple here, and always
# use all columns.
full_idxname = 'index:' + tc.tablename + ':' + idx.idxname
cols = '(' + ','.join([
('c' + str(x)) for x in range(tc.nkeys, tc.nvalues + tc.nkeys)
]) + ')'
self.show_python('# check_entries: ' + full_idxname + cols)
self.show_python("cursor = self.session.open_cursor('" +
full_idxname + cols + "', None, None)")
cursor = self.session.open_cursor(full_idxname + cols, None, None)
count = 0
for x in cursor:
count += 1
self.assertEqual(count, tc.nentries)
for i in range(0, tc.nentries):
key = idx.gen_keys(i)
val = tc.gen_values(i)
self.check_one(full_idxname, cursor, key, val)
cursor.close()
self.show_python("cursor.close()")
class test_tiered10(wttest.WiredTigerTestCase, TieredConfigMixin):
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(), 'test_tiered10', tiered_only=True)
# Make scenarios for different cloud service providers
scenarios = make_scenarios(storage_sources)
# If the 'uri' changes all the other names must change with it.
base = 'test_tiered10-000000000'
obj1file = base + '1.wtobj'
uri = "table:test_tiered10"
conn1_dir = "first_dir"
conn2_dir = "second_dir"
retention = 1
saved_conn = ''
def conn_config(self):
os.mkdir(self.conn1_dir)
os.mkdir(self.conn2_dir)
# Use this to create the directories and set up for the others.
dummy_conn = 'create,statistics=(all),'
# For directory store, the bucket is a directory one level up from database directories.
bucket = ''
if self.ss_name == 'dir_store':
bucket = '../'
bucket += self.bucket
self.saved_conn = get_conn_config(self) + 'bucket=%s,' % bucket + \
'local_retention=%d),create' % self.retention
return dummy_conn
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
def check(self, tc, base, n):
get_check(self, tc, base, n)
# Test calling the flush_tier API.
def test_tiered(self):
# Have two connections running in different directories, but sharing
# the same bucket with different prefixes. Each database creates an
# identically named table with different data. Each then does a flush
# tier testing that both databases can coexist in the same bucket
# without conflict.
#
# Then reopen the connections and make sure we can read data correctly.
#
# We open two connections manually so that they both have the same relative
# pathnames. The standard connection is just a dummy for this test.
ext = self.extensionsConfig()
conn1_params = self.saved_conn + ext + ',tiered_storage=(bucket_prefix=%s)' % self.bucket_prefix
conn1 = self.wiredtiger_open(self.conn1_dir, conn1_params)
session1 = conn1.open_session()
conn2_params = self.saved_conn + ext + ',tiered_storage=(bucket_prefix=%s)' % self.bucket_prefix1
conn2 = self.wiredtiger_open(self.conn2_dir, conn2_params)
session2 = conn2.open_session()
session1.create(self.uri, 'key_format=S,value_format=S,')
session2.create(self.uri, 'key_format=S,value_format=S,')
# Add first data. Checkpoint, flush and close the connection.
c1 = session1.open_cursor(self.uri)
c2 = session2.open_cursor(self.uri)
c1["0"] = "0"
c2["20"] = "20"
self.check(c1, 0, 1)
self.check(c2, 20, 1)
c1.close()
c2.close()
session1.checkpoint()
session1.flush_tier(None)
session2.checkpoint()
session2.flush_tier(None)
conn1_obj1 = os.path.join(self.bucket, self.bucket_prefix + self.obj1file)
conn2_obj1 = os.path.join(self.bucket, self.bucket_prefix1 + self.obj1file)
if self.ss_name == 'dir_store':
self.assertTrue(os.path.exists(conn1_obj1))
self.assertTrue(os.path.exists(conn2_obj1))
conn1.close()
conn2.close()
# Remove the local copies of the objects before we reopen so that we force
# the system to read from the bucket or bucket cache.
local = self.conn1_dir + '/' + self.obj1file
if os.path.exists(local):
os.remove(local)
local = self.conn2_dir + '/' + self.obj1file
if os.path.exists(local):
os.remove(local)
conn1 = self.wiredtiger_open(self.conn1_dir, conn1_params)
session1 = conn1.open_session()
conn2 = self.wiredtiger_open(self.conn2_dir, conn2_params)
session2 = conn2.open_session()
c1 = session1.open_cursor(self.uri)
c2 = session2.open_cursor(self.uri)
self.check(c1, 0, 1)
self.check(c2, 20, 1)
c1.close()
c2.close()
class test_import11(test_import_base):
uri_a = 'table:test_a'
uri_b = 'table:test_b'
bucket = 'bucket1'
cache_bucket = 'cache-bucket1'
nrows = 100
ntables = 10
keys = [b'1', b'2', b'3', b'4', b'5', b'6']
values = [
b'\x01\x02aaa\x03\x04', b'\x01\x02bbb\x03\x04', b'\x01\x02ccc\x03\x04',
b'\x01\x02ddd\x03\x04', b'\x01\x02eee\x03\x04', b'\x01\x02fff\x03\x04'
]
ts = [10 * k for k in range(1, len(keys) + 1)]
create_config = 'allocation_size=512,key_format=u,value_format=u'
tiered_storage_sources = gen_tiered_storage_sources()
scenarios = make_scenarios(tiered_storage_sources)
def conn_config(self):
return self.tiered_conn_config() + ',statistics=(all)'
def get_stat(self, stat):
stat_cursor = self.session.open_cursor('statistics:')
val = stat_cursor[stat][2]
stat_cursor.close()
return val
def create_and_populate(self, uri):
self.session.create(uri, self.create_config)
# Add data and perform a checkpoint.
min_idx = 0
max_idx = len(self.keys) // 3
for i in range(min_idx, max_idx):
self.update(uri, self.keys[i], self.values[i], self.ts[i])
self.checkpoint_and_flush_tier()
# Add more data and checkpoint again.
min_idx = max_idx
max_idx = 2 * len(self.keys) // 3
for i in range(min_idx, max_idx):
self.update(uri, self.keys[i], self.values[i], self.ts[i])
self.checkpoint_and_flush_tier()
return max_idx
def test_file_import(self):
# Create and populate two tables
max_idx = self.create_and_populate(self.uri_a)
max_idx = self.create_and_populate(self.uri_b)
# Create backup export cursor.
c = self.session.open_cursor('backup:export', None, None)
# Copy WiredTiger.export file.
newdir = 'IMPORT_DB'
shutil.rmtree(newdir, ignore_errors=True)
os.mkdir(newdir)
shutil.copy('WiredTiger.export', 'IMPORT_DB')
# Reopen connection in the imported folder.
self.reopen_conn(newdir)
# Make a bunch of files and fill them with data.
self.populate(self.ntables, self.nrows)
self.checkpoint_and_flush_tier()
# Bring forward the oldest to be past or equal to the timestamps we'll be importing.
self.conn.set_timestamp('oldest_timestamp=' +
self.timestamp_str(self.ts[max_idx]))
# Copy over the datafiles for the object we want to import.
self.copy_files('.', newdir)
self.copy_files(self.bucket, os.path.join(newdir, self.bucket))
self.copy_files(self.cache_bucket,
os.path.join(newdir, self.cache_bucket))
# Export the metadata for the current file object 2.
table_config = ""
meta_c = self.session.open_cursor('metadata:', None, None)
for k, v in meta_c:
if k.startswith(self.uri_a):
table_config = cursor[k]
meta_c.close()
# The file_metadata configuration should not be allowed in the tiered storage scenario.
if self.is_tiered_scenario():
msg = "/import for tiered storage is incompatible with the 'file_metadata' setting/"
# Test we cannot use the file_metadata with a tiered table.
invalid_config = 'import=(enabled,repair=false,file_metadata=(' + table_config + '))'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create(self.uri_a, invalid_config), msg)
failed_imports = self.get_stat(
stat.conn.session_table_create_import_fail)
self.assertTrue(failed_imports == 1)
# Test we cannot use the file_metadata with a tiered table and an export file.
invalid_config = 'import=(enabled,repair=false,file_metadata=(' + table_config + '),metadata_file="WiredTiger.export")'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create(self.uri_a, invalid_config), msg)
failed_imports = self.get_stat(
stat.conn.session_table_create_import_fail)
self.assertTrue(failed_imports == 2)
msg = "/Invalid argument/"
# Test importing a tiered table with no import configuration.
invalid_config = 'import=(enabled,repair=false)'
self.assertRaisesWithMessage(
wiredtiger.WiredTigerError,
lambda: self.session.create(self.uri_a, invalid_config), msg)
failed_imports = self.get_stat(
stat.conn.session_table_create_import_fail)
self.assertTrue(failed_imports == 3)
import_config = 'import=(enabled,repair=false,metadata_file="WiredTiger.export")'
# Import the files.
self.session.create(self.uri_a, import_config)
self.checkpoint_and_flush_tier()
# Check the number of files imported after doing an import operation.
files_imported_prev = self.get_stat(
stat.conn.session_table_create_import_success)
self.assertTrue(files_imported_prev == 1)
self.session.create(self.uri_b, import_config)
self.checkpoint_and_flush_tier()
# Check the number of files imported has increased after doing another import operation.
files_imported = self.get_stat(
stat.conn.session_table_create_import_success)
self.assertTrue(files_imported == files_imported_prev + 1)
# Remove WiredTiger.export file.
export_file_path = os.path.join('IMPORT_DB', 'WiredTiger.export')
os.remove(export_file_path)
# FIXME verification is disabled because it fails on tiered storage.
# Verify object.
#self.verifyUntilSuccess(self.session, self.uri_a, None)
# Check that the previously inserted values survived the import.
self.check(self.uri_a, self.keys[:max_idx], self.values[:max_idx])
self.check(self.uri_b, self.keys[:max_idx], self.values[:max_idx])
# Add some data and check that the table operates as usual after importing.
min_idx = max_idx
max_idx = len(self.keys)
for i in range(min_idx, max_idx):
self.update(self.uri_a, self.keys[i], self.values[i], self.ts[i])
self.check(self.uri_a, self.keys, self.values)
# Perform a checkpoint.
self.checkpoint_and_flush_tier()
class test_tiered03(wttest.WiredTigerTestCase, TieredConfigMixin):
K = 1024
M = 1024 * K
G = 1024 * M
# TODO: tiered: change this to a table: URI, otherwise we are
# not using tiered files. The use of a second directory for
# sharing would probably need to be reworked.
uri = 'file:test_tiered03'
storage_sources = gen_tiered_storage_sources(wttest.getss_random_prefix(),
'test_tiered03',
tiered_only=True)
# Occasionally add a lot of records to vary the amount of work flush does.
record_count_scenarios = wtscenario.quick_scenarios(
'nrecs', [10, 10000], [0.9, 0.1])
scenarios = wtscenario.make_scenarios(storage_sources, record_count_scenarios,\
prune=100, prunelong=500)
absolute_bucket_dir = None # initialied in conn_config to an absolute path
def conn_config(self):
bucket_ret = self.bucket
# The bucket format for the S3 store is the name and the region separated by a semi-colon.
if self.ss_name == 's3_store':
cache_dir = self.bucket[:self.bucket.find(';')] + '-cache'
else:
cache_dir = self.bucket + '-cache'
# We have multiple connections that want to share a bucket.
# For the directory store, the first time this function is called, we'll
# establish the absolute path for the bucket, and always use that for
# the bucket name.
# The cache directory name is a relative one, so it won't be shared
# between connections.
if self.ss_name == 'dir_store':
if self.absolute_bucket_dir == None:
self.absolute_bucket_dir = os.path.join(
os.getcwd(), self.bucket)
os.mkdir(self.absolute_bucket_dir)
bucket_ret = self.absolute_bucket_dir
return get_conn_config(self) + 'cache_directory=%s)' % cache_dir
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
# Test sharing data between a primary and a secondary
def test_sharing(self):
# FIXME: WT-8235 Enable the test once file containing transaction ids is supported.
self.skipTest(
'Sharing the checkpoint file containing transaction ids is not supported'
)
ds = SimpleDataSet(self, self.uri, 10)
ds.populate()
ds.check()
self.session.checkpoint()
ds.check()
# Create a secondary database
dir2 = os.path.join(self.home, 'SECONDARY')
os.mkdir(dir2)
conn2 = self.setUpConnectionOpen(dir2)
session2 = conn2.open_session()
# Reference the tree from the secondary:
metac = self.session.open_cursor('metadata:')
metac2 = session2.open_cursor('metadata:', None, 'readonly=0')
uri2 = self.uri[:5] + '../' + self.uri[5:]
metac2[uri2] = metac[self.uri] + ",readonly=1"
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
newds = SimpleDataSet(self, self.uri, 10000)
newds.populate()
newds.check()
self.session.checkpoint()
newds.check()
# Check we can still read from the last checkpoint
cursor2 = session2.open_cursor(uri2)
ds.check_cursor(cursor2)
cursor2.close()
# Bump to new checkpoint
origmeta = metac[self.uri]
checkpoint = re.search(r',checkpoint=\(.+?\)\)', origmeta).group(0)[1:]
self.pr('Orig checkpoint: ' + checkpoint)
session2.alter(uri2, checkpoint)
self.pr('New metadata on secondaery: ' + metac2[uri2])
# Check that we can see the new data
cursor2 = session2.open_cursor(uri2)
newds.check_cursor(cursor2)
class test_tiered09(wttest.WiredTigerTestCase, TieredConfigMixin):
# Make scenarios for different cloud service providers
storage_sources = gen_tiered_storage_sources(
wttest.WiredTigerTestCase._ss_random_prefix,
'test_tiered09',
tiered_only=True)
scenarios = make_scenarios(storage_sources)
# If the 'uri' changes all the other names must change with it.
base = 'test_tiered09-000000000'
base2 = 'test_second09-000000000'
obj1file = base + '1.wtobj'
obj1second = base2 + '1.wtobj'
obj2file = base + '2.wtobj'
uri = "table:test_tiered09"
uri2 = "table:test_second09"
retention = 1
saved_conn = ''
def conn_config(self):
self.saved_conn = get_conn_config(
self) + 'local_retention=%d)' % self.retention
return self.saved_conn
# Load the storage store extension.
def conn_extensions(self, extlist):
TieredConfigMixin.conn_extensions(self, extlist)
def check(self, tc, base, n):
get_check(self, tc, base, n)
# Test calling the flush_tier API.
def test_tiered(self):
# Create a table. Add some data. Checkpoint and flush tier.
# Close the connection. Then we want to reopen the connection
# with a different bucket prefix and repeat. Then reopen the
# connection with the original prefix. Then reopen and verify
# we can read all the data.
#
# Verify the files are as we expect also. We expect:
# 1_<tablename>-00000001.wtobj
# 2_<tablename>-00000002.wtobj
# 1_<tablename>-00000003.wtobj
# but we can read and access all data in all objects.
self.session.create(self.uri, 'key_format=S,value_format=S,')
# Add first data. Checkpoint, flush and close the connection.
c = self.session.open_cursor(self.uri)
c["0"] = "0"
self.check(c, 0, 1)
c.close()
self.session.checkpoint()
self.session.flush_tier(None)
self.close_conn()
# For directory store, check that the expected files exist.
if self.ss_name == 'dir_store':
self.assertTrue(os.path.exists(self.obj2file))
bucket_obj = os.path.join(self.bucket,
self.bucket_prefix + self.obj1file)
self.assertTrue(os.path.exists(bucket_obj))
# Since we've closed and reopened the connection we lost the work units
# to drop the local objects. Clean them up now to make sure we can open
# the correct object in the bucket.
localobj = './' + self.obj1file
if os.path.exists(localobj):
os.remove(localobj)
# Reopen the connection with a different prefix this time.
conn_params = self.saved_conn + ',tiered_storage=(bucket_prefix=%s)' % self.bucket_prefix1
self.conn = self.wiredtiger_open('.', conn_params)
self.session = self.conn.open_session()
# Add a second table created while the second prefix is used for the connection.
self.session.create(self.uri2, 'key_format=S,value_format=S,')
# Add first data. Checkpoint, flush and close the connection.
c = self.session.open_cursor(self.uri2)
c["0"] = "0"
self.check(c, 0, 1)
c.close()
# Add more data to original table.
# Checkpoint, flush and close the connection.
c = self.session.open_cursor(self.uri)
c["1"] = "1"
self.check(c, 0, 2)
c.close()
self.session.checkpoint()
self.session.flush_tier(None)
self.close_conn()
# For directory store, Check each table was created with the correct prefix.
if self.ss_name == 'dir_store':
bucket_obj = os.path.join(self.bucket,
self.bucket_prefix1 + self.obj1second)
self.assertTrue(os.path.exists(bucket_obj))
bucket_obj = os.path.join(self.bucket,
self.bucket_prefix + self.obj2file)
self.assertTrue(os.path.exists(bucket_obj))
# Since we've closed and reopened the connection we lost the work units
# to drop the local objects. Clean them up now to make sure we can open
# the correct object in the bucket.
localobj = './' + self.obj2file
if os.path.exists(localobj):
os.remove(localobj)
localobj = './' + self.obj1second
if os.path.exists(localobj):
os.remove(localobj)
# Reopen with the other prefix and check all data. Even though we're using the
# other prefix, we should find all the data in the object with the original
# prefix.
conn_params = self.saved_conn + ',tiered_storage=(bucket_prefix=%s)' % self.bucket_prefix2
self.conn = self.wiredtiger_open('.', conn_params)
self.session = self.conn.open_session()
c = self.session.open_cursor(self.uri)
self.check(c, 0, 2)
c.close()
c = self.session.open_cursor(self.uri2)
self.check(c, 0, 1)
c.close()
