def test_checkpoint02(self):
done = threading.Event()
self.session.create(
self.uri,
"key_format={},value_format={}".format(self.key_format,
self.value_format))
if self.value_format == '8t':
self.nops *= 2
my_data = 97
else:
my_data = 'a' * self.dsize
ckpt = checkpoint_thread(self.conn, done)
work_queue = queue.Queue()
opthreads = []
try:
ckpt.start()
uris = list()
uris.append(self.uri)
for i in range(self.nops):
if i % 191 == 0 and i != 0:
work_queue.put_nowait(('b', i + 1, my_data))
work_queue.put_nowait(('i', i + 1, my_data))
for i in range(self.nthreads):
t = op_thread(self.conn, uris, self.key_format, work_queue,
done)
opthreads.append(t)
t.start()
except:
# Deplete the work queue if there's an error.
while not work_queue.empty():
work_queue.get()
work_queue.task_done()
raise
finally:
work_queue.join()
done.set()
for t in opthreads:
t.join()
ckpt.join()
# Create a cursor - ensure all items have been put.
cursor = self.session.open_cursor(self.uri, None, None)
i = 1
while True:
nextret = cursor.next()
if nextret != 0:
break
key = cursor.get_key()
value = cursor.get_value()
self.assertEqual(key, i)
self.assertEqual(value, my_data)
i += 1
self.assertEqual(i, self.nops + 1)
def test_backup02(self):
done = threading.Event()
uris = list()
uris.append(self.uri + str(1))
uris.append(self.uri + str(2))
uris.append(self.uri + str(3))
for this_uri in uris:
self.session.create(this_uri,
"key_format=" + self.fmt + ",value_format=S")
# TODO: Ideally we'd like a checkpoint thread, separate to the backup
# thread. Need a way to stop checkpoints while doing backups.
# ckpt = checkpoint_thread(self.conn, done)
# ckpt.start()
bkp = backup_thread(self.conn, 'backup.dir', done)
work_queue = queue.Queue()
opthreads = []
try:
bkp.start()
my_data = 'a' * self.dsize
for i in range(self.nops):
work_queue.put_nowait(('gi', i, my_data))
for i in range(self.nthreads):
t = op_thread(self.conn, uris, self.fmt, work_queue, done)
opthreads.append(t)
t.start()
# Add 200 update entries into the queue every .1 seconds.
more_time = self.time
while more_time > 0:
time.sleep(0.1)
my_data = str(more_time) + 'a' * (self.dsize -
len(str(more_time)))
more_time = more_time - 0.1
for i in range(self.nops):
work_queue.put_nowait(('gu', i, my_data))
except:
# Deplete the work queue if there's an error.
while not work_queue.empty():
work_queue.get()
work_queue.task_done()
raise
finally:
work_queue.join()
done.set()
# Wait for checkpoint thread to notice status change.
# ckpt.join()
for t in opthreads:
t.join()
bkp.join()
def test_checkpoint02(self):
done = threading.Event()
self.session.create(self.uri,
"key_format=" + self.fmt + ",value_format=S")
ckpt = checkpoint_thread(self.conn, done)
ckpt.start()
uris = list()
uris.append(self.uri)
queue = Queue.Queue()
my_data = 'a' * self.dsize
for i in xrange(self.nops):
if i % 191 == 0 and i != 0:
queue.put_nowait(('b', i, my_data))
# if i % 257 == 0 and i != 0:
# queue.put_nowait(('t', i, my_data))
# # Wait another 200 operations, then delete the above table. This
# # not guarantee that the initial operations on the table will have
# # been finished.
# if (i - 100) % 257 == 0 and (i - 100) != 0:
# queue.put_nowait(('d', i - 100, my_data))
queue.put_nowait(('i', i, my_data))
for i in xrange(self.nthreads):
t = op_thread(self.conn, uris, self.fmt, queue, done)
t.start()
queue.join()
done.set()
# Wait for checkpoint thread to notice status change.
while ckpt.is_alive():
time.sleep(0.01)
# Create a cursor - ensure all items have been put.
cursor = self.session.open_cursor(self.uri, None, None)
i = 0
while True:
nextret = cursor.next()
if nextret != 0:
break
key = cursor.get_key()
value = cursor.get_value()
self.assertEqual(key, i)
self.assertEqual(value, my_data)
i += 1
self.assertEqual(i, self.nops)
def test_concurrent_operations_with_backup(self):
done = threading.Event()
table_uri = 'table:' + self.uri
# Create and populate the table.
self.session.create(table_uri, "key_format=S,value_format=S")
self.add_data(table_uri, 'key', 'value', True)
work_queue = queue.Queue()
t = op_thread(self.conn, [table_uri], self.key_fmt, work_queue, done)
try:
t.start()
# Place create or drop operation into work queue.
iteration = 0
op = 't'
for _ in range(0, self.ops):
# Open backup cursor.
bkup_c = self.session.open_cursor('backup:', None, None)
work_queue.put_nowait((op, str(iteration), 'value'))
all_files = self.take_full_backup(self.dir, bkup_c)
if op == 't':
# Newly created table shouldn't be present in backup.
self.assertTrue(self.uri + str(iteration) +
".wt" not in all_files)
iteration = iteration + 1
else:
# Dropped table should still be present in backup.
self.assertTrue(self.uri + str(iteration) +
".wt" in all_files)
iteration = iteration + 1
bkup_c.close()
# Once we reach midway point, start drop operations.
if iteration == self.ops / 2:
iteration = 0
op = 'd'
except:
# Deplete the work queue if there's an error.
while not work_queue.empty():
work_queue.get()
work_queue.task_done()
raise
finally:
work_queue.join()
done.set()
t.join()
def test_backup02(self):
done = threading.Event()
uris = list()
uris.append(self.uri + str(1))
uris.append(self.uri + str(2))
uris.append(self.uri + str(3))
for this_uri in uris:
self.session.create(this_uri,
"key_format=" + self.fmt + ",value_format=S")
# TODO: Ideally we'd like a checkpoint thread, separate to the backup
# thread. Need a way to stop checkpoints while doing backups.
# ckpt = checkpoint_thread(self.conn, done)
# ckpt.start()
bkp = backup_thread(self.conn, 'backup.dir', done)
bkp.start()
queue = Queue.Queue()
my_data = 'a' * self.dsize
for i in xrange(self.nops):
queue.put_nowait(('gi', i, my_data))
opthreads = []
for i in xrange(self.nthreads):
t = op_thread(self.conn, uris, self.fmt, queue, done)
opthreads.append(t)
t.start()
# Add 200 update entries into the queue every .1 seconds.
more_time = self.time
while more_time > 0:
time.sleep(0.1)
my_data = str(more_time) + 'a' * (self.dsize - len(str(more_time)))
more_time = more_time - 0.1
for i in xrange(self.nops):
queue.put_nowait(('gu', i, my_data))
queue.join()
done.set()
# # Wait for checkpoint thread to notice status change.
# ckpt.join()
for t in opthreads:
t.join()
bkp.join()
def test_checkpoint02(self):
done = threading.Event()
self.session.create(self.uri,
"key_format=" + self.fmt + ",value_format=S")
ckpt = checkpoint_thread(self.conn, done)
ckpt.start()
uris = list()
uris.append(self.uri)
queue = Queue.Queue()
my_data = 'a' * self.dsize
for i in xrange(self.nops):
if i % 191 == 0 and i != 0:
queue.put_nowait(('b', i, my_data))
queue.put_nowait(('i', i, my_data))
opthreads = []
for i in xrange(self.nthreads):
t = op_thread(self.conn, uris, self.fmt, queue, done)
opthreads.append(t)
t.start()
queue.join()
done.set()
for t in opthreads:
t.join()
ckpt.join()
# Create a cursor - ensure all items have been put.
cursor = self.session.open_cursor(self.uri, None, None)
i = 0
while True:
nextret = cursor.next()
if nextret != 0:
break
key = cursor.get_key()
value = cursor.get_value()
self.assertEqual(key, i)
self.assertEqual(value, my_data)
i += 1
self.assertEqual(i, self.nops)