def test_unordered_key_generator(self):
ws = WorkloadSettings(items=10**5,
workers=25,
working_set=100,
working_set_access=100,
working_set_moving_docs=0)
keys = []
for worker in range(ws.workers):
generator = docgen.UnorderedKey(worker, ws, prefix='test')
keys += [key for key in generator]
expected = ['test-%012d' % i for i in range(ws.items)]
self.assertEqual(sorted(keys), expected)
def test_key_for_removal(self):
ws = WorkloadSettings(items=10**3,
workers=20,
working_set=100,
working_set_access=100,
working_set_moving_docs=0)
keys = set()
for worker in range(ws.workers):
for key in docgen.UnorderedKey(sid=worker, ws=ws, prefix='test'):
keys.add(key)
key_gen = docgen.KeyForRemoval(prefix='test')
for op in range(1, 100):
key = key_gen.next(op)
self.assertIn(key, keys)
def test_working_set_keys(self):
ws = WorkloadSettings(items=10**3,
workers=10,
working_set=90,
working_set_access=50,
working_set_moving_docs=0)
keys = set()
for worker in range(ws.workers):
for key in docgen.UnorderedKey(sid=worker, ws=ws, prefix='test'):
keys.add(key)
key_gen = docgen.WorkingSetKey(ws=ws, prefix='test')
for op in range(10**4):
key = key_gen.next(curr_items=ws.items, curr_deletes=0)
self.assertIn(key, keys)
def test_zipf_generator(self):
ws = WorkloadSettings(items=10**3,
workers=40,
working_set=10,
working_set_access=100,
working_set_moving_docs=0)
keys = set()
for worker in range(ws.workers):
for key in docgen.UnorderedKey(sid=worker, ws=ws, prefix='test'):
self.assertNotIn(key, keys)
keys.add(key)
self.assertEqual(len(keys), ws.items)
key_gen = docgen.ZipfKey(prefix='test')
for op in range(10**4):
key = key_gen.next(curr_deletes=100, curr_items=ws.items)
self.assertIn(key, keys)
def test_cas_updates(self):
ws = WorkloadSettings(items=10**3,
workers=20,
working_set=100,
working_set_access=100,
working_set_moving_docs=0)
keys = set()
for worker in range(ws.workers):
for key in docgen.UnorderedKey(sid=worker, ws=ws, prefix='test'):
keys.add(key)
cases = defaultdict(set)
key_gen = docgen.KeyForCASUpdate(total_workers=ws.workers,
prefix='test')
for sid in 5, 6:
for op in range(10**3):
key = key_gen.next(sid=sid, curr_items=ws.items)
self.assertIn(key, keys)
cases[sid].add(key)
def test_hot_keys(self):
ws = WorkloadSettings(items=10**4,
workers=40,
working_set=10,
working_set_access=100,
working_set_moving_docs=0)
keys = set()
for worker in range(ws.workers):
for key in docgen.UnorderedKey(sid=worker, ws=ws, prefix='test'):
self.assertNotIn(key, keys)
keys.add(key)
self.assertEqual(len(keys), ws.items)
hot_keys = set()
for worker in range(ws.workers):
for key in docgen.HotKey(sid=worker, ws=ws, prefix='test'):
self.assertNotIn(key, hot_keys)
self.assertIn(key, keys)
hot_keys.add(key)
self.assertEqual(len(hot_keys), ws.working_set * ws.items // 100)