def test_small_documents(self):
key_gen = docgen.NewOrderedKey(prefix='test', fmtr='decimal')
doc_gen = docgen.Document(avg_size=150)
for i in range(10 ** 3):
key = key_gen.next(i)
doc = doc_gen.next(key=key)
size = len(str(doc))
self.assertEqual(doc["body"], "")
self.assertAlmostEqual(size, doc_gen.OVERHEAD, delta=20)
def test_doc_size_variation(self):
size = 512
key_gen = docgen.NewOrderedKey(prefix='test', fmtr='decimal')
doc_gen = docgen.Document(avg_size=size)
for i in range(10 ** 4):
key = key_gen.next(i)
doc = doc_gen.next(key=key)
actual_size = len(str(doc))
self.assertAlmostEqual(actual_size, size,
delta=size * doc_gen.SIZE_VARIATION)
def test_doc_size(self):
size = 1024
key_gen = docgen.NewOrderedKey(prefix='n1ql', fmtr='decimal')
for dg in self.doc_generators(size=size):
for i in range(10 ** 4):
key = key_gen.next(i)
doc = dg.next(key=key)
actual_size = len(str(doc))
self.assertAlmostEqual(actual_size, size,
delta=size * 0.05, # 5% variation
msg=dg.__class__.__name__)
def test_compression_ratio(self):
size = 1024
key_gen = docgen.NewOrderedKey(prefix='test', fmtr='decimal')
doc_gen = docgen.LargeDocument(avg_size=size)
for i in range(10 ** 4):
key = key_gen.next(i)
doc = doc_gen.next(key)
value = json.dumps(doc)
compressed = snappy.compress(value)
ratio = len(value) / len(compressed)
self.assertLess(ratio, 1.75, value)
def test_large_documents(self):
size = 1024
key_gen = docgen.NewOrderedKey(prefix='test', fmtr='decimal')
doc_gen = docgen.LargeDocument(avg_size=size)
for i in range(10 ** 4):
key = key_gen.next(i)
doc = doc_gen.next(key=key)
value = json.dumps(doc)
actual_size = len(value)
self.assertAlmostEqual(actual_size, size,
delta=size * doc_gen.SIZE_VARIATION,
msg=value)
def test_new_ordered_keys(self):
ws = WorkloadSettings(items=10 ** 4, workers=40, working_set=10,
working_set_access=100, working_set_moving_docs=0,
key_fmtr='decimal')
keys = set()
for worker in range(ws.workers):
for key in docgen.SequentialKey(sid=worker, ws=ws, prefix='test'):
keys.add(key)
key_gen = docgen.NewOrderedKey(prefix='test', fmtr='decimal')
for op in range(1, 10 ** 3):
key = key_gen.next(ws.items + op)
self.assertNotIn(key, keys)
