def test_jsonload_util(self):
# LSM and column-store isn't a valid combination.
if self.skip():
return
# Create the object.
uri = self.uri + self.name
uri2 = self.uri + self.name2
ds = self.dataset(self, uri, self.nentries, key_format=self.keyfmt)
ds.populate()
# Dump the object.
self.runWt(['dump', '-j', uri], outfilename='jsondump.out')
loadcmd = ['load', '-jf', 'jsondump.out', '-r', self.name2]
if self.keyfmt == 'r':
loadcmd.append('-a')
self.runWt(loadcmd)
# Check the contents of the data we read.
# We use the dataset only for checking.
ds2 = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt)
ds2.check()
# Reload into the original uri, and dump into another file.
self.session.drop(uri, None)
self.session.drop(uri2, None)
self.runWt(['load', '-jf', 'jsondump.out'])
self.runWt(['dump', '-j', uri], outfilename='jsondump2.out')
# Compare the two outputs, and check the content again.
compare_files(self, 'jsondump.out', 'jsondump2.out')
ds.check()
def test_jsonload_util(self):
# LSM and column-store isn't a valid combination.
if self.skip():
return
# Create the object.
uri = self.uri + self.name
uri2 = self.uri + self.name2
ds = self.dataset(self, uri, self.nentries, key_format=self.keyfmt)
ds.populate()
# Dump the object.
self.runWt(['dump', '-j', uri], outfilename='jsondump.out')
loadcmd = ['load', '-jf', 'jsondump.out', '-r', self.name2]
if self.keyfmt == 'r':
loadcmd.append('-a')
self.runWt(loadcmd)
# Check the contents of the data we read.
# We use the dataset only for checking.
ds2 = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt)
ds2.check()
# Reload into the original uri, and dump into another file.
self.session.drop(uri, None)
self.session.drop(uri2, None)
self.runWt(['load', '-jf', 'jsondump.out'])
self.runWt(['dump', '-j', uri], outfilename='jsondump2.out')
# Compare the two outputs, and check the content again.
compare_files(self, 'jsondump.out', 'jsondump2.out')
ds.check()
def test_backup_database(self):
self.populate()
os.mkdir(self.dir)
self.runWt(['backup', self.dir])
# Make sure all the files were copied.
self.runWt(['list'], outfilename='outfile.orig')
self.runWt(['-h', self.dir, 'list'], outfilename='outfile.backup')
compare_files(self, 'outfile.orig', 'outfile.backup')
# And that the contents are the same.
for i in self.objs:
self.compare(i[0])
def test_dump_obsolete_data(self):
uri = 'table:test_util21'
create_params = 'key_format=S,value_format=S'
self.session.create(uri, create_params)
value1 = 'a' * 100
value2 = 'b' * 100
value3 = 'c' * 100
value4 = 'd' * 100
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(1))
self.add_data_with_timestamp(uri, value1, 2)
self.add_data_with_timestamp(uri, value2, 3)
self.add_data_with_timestamp(uri, value3, 5)
self.add_data_with_timestamp(uri, value4, 7)
# Perform checkpoint, to clean the dirty pages and place values on disk.
self.session.checkpoint()
# Set stable timestamp, so we don't lose data when closing/opening connection when using wt dump.
self.conn.set_timestamp('stable_timestamp=' + timestamp_str(10))
# Call dump on the values before the oldest timestamp is set
self.runWt(['dump', 'file:WiredTigerHS.wt'],
outfilename="before_oldest")
# Set oldest timestamp, and checkpoint, the obsolete data should not removed as
# the pages are clean.
self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(6))
self.session.checkpoint()
self.runWt(['dump', 'file:WiredTigerHS.wt'],
outfilename="after_oldest")
self.assertEqual(True,
compare_files(self, "before_oldest", "after_oldest"))
def test_backup_database(self):
self.populate(self.objs)
os.mkdir(self.dir)
self.runWt(['backup', self.dir])
# Make sure all the files were copied.
self.runWt(['list'], outfilename='outfile.orig')
self.runWt(['-h', self.dir, 'list'], outfilename='outfile.backup')
self.assertEqual(True,
compare_files(self, 'outfile.orig', 'outfile.backup'))
# And that the contents are the same.
for i in self.objs:
self.compare_backups(i[0], self.dir, './')
def test_backup_database(self):
self.populate(0)
os.mkdir(self.dir)
self.runWt(['backup', self.dir])
# Make sure all the files were copied.
self.runWt(['list'], outfilename='outfile.orig')
# Pass -R flag as we need to run recovery on the backup dir before calling list.
self.runWt(['-h', self.dir, '-R', 'list'], outfilename='outfile.backup')
self.assertEqual(
True, compare_files(self, 'outfile.orig', 'outfile.backup'))
# And that the contents are the same.
for i in self.objs:
self.compare(i[0])
def compare_backups(self, t_uri):
# Run wt dump on full backup directory.
full_backup_out = self.full_out + '.' + str(self.counter)
home_dir = self.home_full + '.' + str(self.counter)
if self.counter == 0:
home_dir = self.home
self.runWt(['-R', '-h', home_dir, 'dump', t_uri],
outfilename=full_backup_out)
# Run wt dump on incremental backup directory.
incr_backup_out = self.incr_out + '.' + str(self.counter)
home_dir = self.home_incr + '.' + str(self.counter)
self.runWt(['-R', '-h', home_dir, 'dump', t_uri],
outfilename=incr_backup_out)
self.assertEqual(True,
compare_files(self, full_backup_out, incr_backup_out))
def compare_backups(self, uri, base_dir, other_dir, suffix=None):
sfx = ""
if suffix != None:
sfx = "." + suffix
base_out = "./backup_base" + sfx
if os.path.exists(base_out):
os.remove(base_out)
# Run wt dump on base backup directory
self.runWt(['-R', '-h', base_dir, 'dump', uri], outfilename=base_out)
other_out = "./backup_other" + sfx
if os.path.exists(other_out):
os.remove(other_out)
# Run wt dump on incremental backup
self.runWt(['-R', '-h', other_dir, 'dump', uri], outfilename=other_out)
self.pr("compare_files: " + base_out + ", " + other_out)
self.assertEqual(True, compare_files(self, base_out, other_out))
def compare(self, uri, dir_full, dir_incr):
#print "Compare: full URI: " + uri_full + " with incremental URI: " + uri_incr
if dir_full == None:
full_name = 'original'
else:
full_name = 'backup_full'
incr_name = 'backup_incr'
if os.path.exists(full_name):
os.remove(full_name)
if os.path.exists(incr_name):
os.remove(incr_name)
if dir_full == None:
self.runWt(['dump', uri], outfilename=full_name)
else:
self.runWt(['-h', dir_full, 'dump', uri], outfilename=full_name)
self.runWt(['-h', dir_incr, 'dump', uri], outfilename=incr_name)
self.assertEqual(True, compare_files(self, full_name, incr_name))
def compare(self, uri, dir_full, dir_incr):
# print "Compare: full URI: " + uri + " with incremental URI "
if dir_full == None:
full_name='original'
else:
full_name='backup_full'
incr_name='backup_incr'
if os.path.exists(full_name):
os.remove(full_name)
if os.path.exists(incr_name):
os.remove(incr_name)
#
# We have been copying the logs only, so we need to force 'wt' to
# run recovery in order to apply all the logs and check the data.
#
if dir_full == None:
self.runWt(['-R', 'dump', uri], outfilename=full_name)
else:
self.runWt(['-R', '-h', dir_full, 'dump', uri], outfilename=full_name)
self.runWt(['-R', '-h', dir_incr, 'dump', uri], outfilename=incr_name)
self.assertEqual(True,
compare_files(self, full_name, incr_name))
def compare(self, uri):
self.runWt(['dump', uri], outfilename='orig')
self.runWt(['-h', self.dir, 'dump', uri], outfilename='backup')
self.assertEqual(True, compare_files(self, 'orig', 'backup'))
def compare(self, uri):
self.runWt(['dump', uri], outfilename='orig')
# Pass -R flag as we need to run recovery on the backup dir before calling dump.
self.runWt(['-h', self.dir, '-R', 'dump', uri], outfilename='backup')
self.assertEqual(True, compare_files(self, 'orig', 'backup'))