def file_ls_ops(self, c):
write_offset = 0
for i in range(0, 1000):
try:
c.write_ls(b(str(i)))
except KeyError as k:
self.fail('Received error message: {}'.format(k))
read_offset = 0
for i in range(0, 1000):
c.seek(read_offset)
self.assertEqual(b(str(i)), c.read_ls(len(b(str(i)))))
read_offset += len(b(str(i)))
for i in range(1000, 2000):
c.seek(read_offset)
self.assertRaises(KeyError, c.read_ls, len(b(str(i))))
read_offset += len(b(str(i)))
read_offset = 0
for i in range(0, 1000):
c.seek(read_offset)
self.assertEqual(b(str(i)), c.read_ls(len(b(str(i)))))
read_offset += len(b(str(i)))
def test_notifications(self):
self.start_servers()
client = self.jiffy_client()
try:
client.create_hash_table("/a/file.txt", "local://tmp")
n1 = client.listen("/a/file.txt")
n2 = client.listen("/a/file.txt")
n3 = client.listen("/a/file.txt")
n1.subscribe(['put'])
n2.subscribe(['put', 'remove'])
n3.subscribe(['remove'])
kv = client.open_hash_table("/a/file.txt")
kv.put(b'key1', b'value1')
kv.remove(b'key1')
self.assertEqual(Notification('put', b('key1')), n1.get_notification())
n2_notifs = [n2.get_notification(), n2.get_notification()]
self.assertTrue(Notification('put', b('key1')) in n2_notifs)
self.assertTrue(Notification('remove', b('key1')) in n2_notifs)
self.assertEqual(Notification('remove', b('key1')), n3.get_notification())
with self.assertRaises(queue.Empty):
n1.get_notification(block=False)
with self.assertRaises(queue.Empty):
n2.get_notification(block=False)
with self.assertRaises(queue.Empty):
n3.get_notification(block=False)
n1.unsubscribe(['put'])
n2.unsubscribe(['remove'])
kv.put(b'key1', b'value1')
kv.remove(b'key1')
self.assertEqual(Notification('put', b'key1'), n2.get_notification())
self.assertEqual(Notification('remove', b'key1'), n3.get_notification())
with self.assertRaises(queue.Empty):
n1.get_notification(block=False)
with self.assertRaises(queue.Empty):
n2.get_notification(block=False)
with self.assertRaises(queue.Empty):
n3.get_notification(block=False)
n1.disconnect()
n2.disconnect()
n3.disconnect()
finally:
client.disconnect()
self.stop_servers()
def queue_ls_ops(self, q):
length = 0
for i in range(0, 1000):
try:
q.put_ls(b(str(i)))
except KeyError as k:
self.fail('Received error message: {}'.format(k))
for i in range(0, 1000):
self.assertEqual(b(str(i)), q.get_ls())
for i in range(1000, 2000):
self.assertRaises(KeyError, q.get_ls)
def test_file_asyn_read_write(self):
self.start_servers(auto_scale=True)
client = self.jiffy_client()
data_size = 25000
data = b("")
for i in range(0, data_size):
data += b("a")
file_client = client.create_file("/a/file.txt", "local://tmp")
self.assertEqual(data_size, file_client.write(data))
self.assertEqual(4, len(client.fs.dstatus("/a/file.txt").data_blocks))
self.assertTrue(file_client.seek(0))
self.assertEqual(data, file_client.read(len(data)))
self.assertEqual(4, len(client.fs.dstatus("/a/file.txt").data_blocks))
client.disconnect()
self.stop_servers()
def test_fifo_queue_auto_scale(self):
self.start_servers(auto_scale=True)
client = self.jiffy_client()
try:
queue_client = client.create_queue("/a/file.txt", "local://tmp")
for i in range(0, 1000):
try:
queue_client.put(b(str(i)))
except KeyError as k:
self.fail("Received error message: {}".format(k))
self.assertEqual(2, len(client.fs.dstatus("/a/file.txt").data_blocks))
for i in range(0, 1000):
self.assertEqual(b(str(i)), queue_client.get())
self.assertEqual(1, len(client.fs.dstatus("/a/file.txt").data_blocks))
finally:
client.disconnect()
self.stop_servers()
def shared_log_ops(self, c):
for i in range(0, 1000):
try:
c.write(i, str(i)+"_data", [str(i)+"_stream"])
except KeyError as k:
self.fail('Received error message: {}'.format(k))
for i in range(0, 1000):
self.assertEqual([b(str(i)+"_data")], c.scan(i, i, [str(i)+"_stream"]))
def test_large_data_read(self):
self.start_servers()
client = self.jiffy_client()
try:
c = client.create_file('/a/file.txt', 'local://tmp')
self.assertTrue(client.fs.exists('/a/file.txt'))
data = b("")
data_size = 1024 * 1024
for i in range(0, data_size):
data += b("a")
for i in range(0, 50):
try:
c.write(data)
except KeyError as k:
self.fail('Received error message: {}'.format(k))
self.assertTrue(c.seek(0))
for i in range(0, 50):
self.assertEqual(data, c.read(len(data)))
finally:
client.disconnect()
self.stop_servers()
def queue_ops(self, q):
length = 0
for i in range(0, 1000):
try:
q.put(b(str(i)))
length += len(b(str(i)))
except KeyError as k:
self.fail('Received error message: {}'.format(k))
self.assertEqual(length, q.length())
q.in_rate()
q.out_rate()
for i in range(0, 1000):
self.assertEqual(b(str(i)), q.read_next())
for i in range(0, 1000):
self.assertEqual(b(str(i)), q.get())
q.in_rate()
q.out_rate()
self.assertEqual(0, q.length())
for i in range(1000, 2000):
self.assertRaises(KeyError, q.get)
def queue_worker(self, client, query_type, result_queue, ops):
q = client.open_queue('/a/file.txt')
if query_type == QueryType.PUT_QUERY:
for i in range(0, ops):
try:
q.put(b(str(i)))
except KeyError as k:
self.fail('Received error message: {}'.format(k))
elif query_type == QueryType.GET_QUERY:
for i in range(0, ops * 10):
try:
q.get()
result_queue.put(1)
except KeyError as k:
continue
def file_ops(self, c):
for i in range(0, 1000):
try:
c.write(b(str(i)))
except KeyError as k:
self.fail('Received error message: {}'.format(k))
self.assertTrue(c.seek(0))
for i in range(0, 1000):
self.assertEqual(b(str(i)), c.read(len(b(str(i)))))
for i in range(1000, 2000):
self.assertRaises(KeyError, c.read, len(b(str(i))))
self.assertTrue(c.seek(0))
for i in range(0, 1000):
self.assertEqual(b(str(i)), c.read(len(b(str(i)))))
def hash_table_ls_ops(self, kv):
if getattr(kv, "num_keys", None) is not None:
self.assertEqual(0, kv.num_keys())
# Test exists/get/put
for i in range(0, 1000):
try:
kv.put_ls(b(str(i)), b(str(i)))
except KeyError as k:
self.fail("Received error message: {}".format(k))
for i in range(0, 1000):
self.assertTrue(kv.exists_ls(b(str(i))))
self.assertEqual(b(str(i)), kv.get_ls(b(str(i))))
for i in range(1000, 2000):
self.assertFalse(kv.exists_ls(b(str(i))))
self.assertRaises(KeyError, kv.get_ls, b(str(i)))
# Test update
for i in range(0, 1000):
self.assertEqual(b('!ok'), kv.update_ls(b(str(i)), b(str(i + 1000))))
for i in range(1000, 2000):
self.assertRaises(KeyError, kv.update_ls, b(str(i)), b(str(i + 1000)))
for i in range(0, 1000):
self.assertEqual(b(str(i + 1000)), kv.get_ls(b(str(i))))
# Test remove
for i in range(0, 1000):
self.assertEqual(b("!ok"), kv.remove_ls(b(str(i))))
for i in range(1000, 2000):
self.assertRaises(KeyError, kv.remove_ls, b(str(i)))
for i in range(0, 1000):
self.assertRaises(KeyError, kv.get_ls, b(str(i)))