def test_achievements_length(self):
docgen = NewNestedDocument(avg_size=0)
for k in range(100000):
key = '%012d' % k
doc = docgen.next(key)
self.assertLessEqual(len(doc['achievements']), 10)
self.assertGreater(len(doc['achievements']), 0)
def test_query_formatting(self):
docgen = NewNestedDocument(avg_size=self.SIZE)
doc = docgen.next('test-key')
queries = ['SELECT * from `bucket-1`;', 'SELECT count(*) from `bucket-1`;']
qgen = N1QLQueryGen(queries=queries)
_, _, query = qgen.next(doc)
query.format(bucket='bucket-1')
def test_achievements_length(self):
docgen = NewNestedDocument(avg_size=0)
for k in range(100000):
key = '%012d' % k
doc = docgen.next(key)
self.assertLessEqual(len(doc['achievements']), 10)
self.assertGreater(len(doc['achievements']), 0)
def test_gmtime_variation(self):
docgen = NewNestedDocument(avg_size=0)
keys = set()
for k in range(1000):
key = '%012d' % k
doc = docgen.next(key)
keys.add(doc['gmtime'])
self.assertEqual(len(keys), 12)
def test_gmtime_variation(self):
docgen = NewNestedDocument(avg_size=0)
keys = set()
for k in range(1000):
key = '%012d' % k
doc = docgen.next(key)
keys.add(doc['gmtime'])
self.assertEqual(len(keys), 12)
def test_query_formatting(self):
docgen = NewNestedDocument(avg_size=self.SIZE)
doc = docgen.next('test-key')
queries = [
'SELECT * from `bucket-1`;', 'SELECT count(*) from `bucket-1`;'
]
qgen = N1QLQueryGen(queries=queries)
_, _, query = qgen.next(doc)
query.format(bucket='bucket-1')
def test_doc_content(self):
docgen = NewNestedDocument(avg_size=0)
actual = docgen.next(key='000000000020')
expected = {
'name': {
'f': {
'f': {
'f': 'ecdb3e e921c9'
}
}
},
'email': {
'f': {
'f': '*****@*****.**'
}
},
'street': {
'f': {
'f': '400f1d0a'
}
},
'city': {
'f': {
'f': '90ac48'
}
},
'county': {
'f': {
'f': '40efd6'
}
},
'state': {
'f': 'WY'
},
'full_state': {
'f': 'Montana'
},
'country': {
'f': '1811db'
},
'realm': {
'f': '15e3f5'
},
'coins': {
'f': 213.54
},
'category': 1,
'achievements': [0, 135, 92],
'gmtime': (1972, 3, 3, 0, 0, 0, 4, 63, 0),
'year': 1989,
'body': '',
'capped_small': '100_0',
'capped_large': '3000_0',
}
self.assertEqual(actual, expected)
def test_doc_size(self):
docgen = NewNestedDocument(avg_size=self.SIZE)
sizes = tuple(
len(json.dumps(docgen.next(key='%012s' % i)))
for i in range(10000))
mean = np.mean(sizes)
self.assertAlmostEqual(mean, 1152, delta=128)
p95 = np.percentile(sizes, 97)
self.assertLess(p95, 2048)
p99 = np.percentile(sizes, 98)
self.assertGreater(p99, 2048)
self.assertLess(max(sizes), 2 * 1024**2)
self.assertGreater(min(sizes), 0)
def test_doc_size(self):
docgen = NewNestedDocument(avg_size=self.SIZE)
sizes = tuple(
len(json.dumps(docgen.next(key='%012s' % i)))
for i in range(10000)
)
mean = np.mean(sizes)
self.assertAlmostEqual(mean, 1024, delta=128)
p95 = np.percentile(sizes, 97)
self.assertLess(p95, 2048)
p99 = np.percentile(sizes, 98)
self.assertGreater(p99, 2048)
self.assertLess(max(sizes), 2 * 1024 ** 2)
self.assertGreater(min(sizes), 0)
def __init__(self, settings, workload, prefix=None):
super(Latency, self).__init__(settings)
self.clients = []
for bucket in self.get_buckets():
client = CBGen(bucket=bucket, host=settings.master_node,
username=bucket, password=settings.bucket_password)
self.clients.append((bucket, client))
self.existing_keys = ExistingKey(workload.working_set,
workload.working_set_access,
prefix=prefix)
if not hasattr(workload, 'doc_gen') or workload.doc_gen == 'old':
self.new_docs = NewDocument(workload.size)
else:
self.new_docs = NewNestedDocument(workload.size)
self.items = workload.items
def __init__(self, settings, workload, prefix=None):
super(Latency, self).__init__(settings)
self.clients = []
for bucket in self.get_buckets():
client = CBGen(bucket=bucket,
host=settings.master_node,
username=bucket,
password=settings.bucket_password)
self.clients.append((bucket, client))
self.existing_keys = ExistingKey(workload.working_set,
workload.working_set_access,
prefix=prefix)
self.new_keys = NewKey(prefix=prefix, expiration=workload.expiration)
self.keys_for_removal = KeyForRemoval(prefix=prefix)
if not hasattr(workload, 'doc_gen') or workload.doc_gen == 'old':
self.new_docs = NewDocument(workload.size)
elif workload.doc_gen == 'new':
self.new_docs = NewNestedDocument(workload.size)
elif workload.doc_gen == 'reverse_lookup':
self.new_docs = ReverseLookupDocument(workload.size,
workload.doc_partitions)
elif workload.doc_gen == 'reverse_lookup_array_indexing':
self.new_docs = ReverseLookupDocumentArrayIndexing(
workload.size, workload.doc_partitions, workload.items)
self.items = workload.items
self.n1ql_op = workload.n1ql_op
def test_doc_content(self):
docgen = NewNestedDocument(avg_size=0)
actual = docgen.next(key='000000000020')
expected = {
'name': {'f': {'f': {'f': 'ecdb3e e921c9'}}},
'email': {'f': {'f': '*****@*****.**'}},
'street': {'f': {'f': '400f1d0a'}},
'city': {'f': {'f': '90ac48'}},
'county': {'f': {'f': '40efd6'}},
'state': {'f': 'WY'},
'full_state': {'f': 'Montana'},
'country': {'f': '1811db'},
'realm': {'f': '15e3f5'},
'coins': {'f': 213.54},
'category': 1,
'achievements': [0, 135, 92],
'gmtime': (1972, 3, 3, 0, 0, 0, 4, 63, 0),
'year': 1989,
'body': '',
}
self.assertEqual(actual, expected)
class SpringLatency(Latency):
COLLECTOR = "spring_latency"
METRICS = ("latency_set", "latency_get")
def __init__(self, settings, workload, prefix=None):
super(Latency, self).__init__(settings)
self.clients = []
for bucket in self.get_buckets():
client = CBGen(bucket=bucket, host=settings.master_node,
username=bucket, password=settings.bucket_password)
self.clients.append((bucket, client))
self.existing_keys = ExistingKey(workload.working_set,
workload.working_set_access,
prefix=prefix)
if not hasattr(workload, 'doc_gen') or workload.doc_gen == 'old':
self.new_docs = NewDocument(workload.size)
else:
self.new_docs = NewNestedDocument(workload.size)
self.items = workload.items
def measure(self, client, metric):
key = self.existing_keys.next(curr_items=self.items, curr_deletes=0)
doc = self.new_docs.next(key)
t0 = time()
if metric == "latency_set":
client.create(key, doc)
elif metric == "latency_get":
client.read(key)
elif metric == "latency_cas":
client.cas(key, doc)
return 1000 * (time() - t0) # Latency in ms
def sample(self):
for bucket, client in self.clients:
samples = {}
for metric in self.METRICS:
samples[metric] = self.measure(client, metric)
self.store.append(samples, cluster=self.cluster,
bucket=bucket, collector=self.COLLECTOR)
def run(keys):
docs = NewNestedDocument(avg_size=1024)
for key in keys:
docs.next(key)
def test_determenistic(self):
docgen = NewNestedDocument(avg_size=self.SIZE)
d1 = docgen.next(key='mykey')
d2 = docgen.next(key='mykey')
d1['body'] = d2['body'] = None
self.assertEqual(d1, d2)
def test_alphabet_size(self):
docgen = NewNestedDocument(avg_size=self.SIZE)
alphabet = docgen._build_alphabet('key')
self.assertEqual(len(alphabet), 64)
def __init__(self, workload_settings, target_settings,
shutdown_event=None):
self.ws = workload_settings
self.ts = target_settings
self.shutdown_event = shutdown_event
logger.setLevel(logging.INFO)
self.existing_keys = ExistingKey(self.ws.working_set,
self.ws.working_set_access,
self.ts.prefix)
self.new_keys = NewKey(self.ts.prefix, self.ws.expiration)
self.keys_for_removal = KeyForRemoval(self.ts.prefix)
if not hasattr(self.ws, 'doc_gen') or self.ws.doc_gen == 'old':
extra_fields = False
if (hasattr(self.ws, 'extra_doc_fields') and
self.ws['extra_doc_fields'] == 'yes'):
extra_fields = True
self.docs = NewDocument(self.ws.size, extra_fields)
elif self.ws.doc_gen == 'new':
self.docs = NewNestedDocument(self.ws.size)
elif self.ws.doc_gen == 'merge':
isRandom = True
if self.ts.prefix == 'n1ql':
isRandom = False
self.docs = MergeDocument(self.ws.size,
self.ws.doc_partitions,
isRandom)
elif self.ws.doc_gen == 'reverse_lookup':
isRandom = True
if self.ts.prefix == 'n1ql':
isRandom = False
self.docs = ReverseLookupDocument(self.ws.size,
self.ws.doc_partitions,
isRandom)
elif self.ws.doc_gen == 'reverse_lookup_array_indexing':
isRandom = True
if self.ts.prefix == 'n1ql':
isRandom = False
if self.ws.updates:
# plus 10 to all values in array when updating doc
self.docs = ReverseLookupDocumentArrayIndexing(
self.ws.size, self.ws.doc_partitions, self.ws.items,
delta=random.randint(0, 10))
else:
self.docs = ReverseLookupDocumentArrayIndexing(
self.ws.size, self.ws.doc_partitions, self.ws.items)
elif self.ws.doc_gen == 'spatial':
self.docs = NewDocumentFromSpatialFile(
self.ws.spatial.data,
self.ws.spatial.dimensionality)
elif self.ws.doc_gen == 'large_subdoc':
self.docs = NewLargeDocument(self.ws.size)
self.next_report = 0.05 # report after every 5% of completion
host, port = self.ts.node.split(':')
# Only FTS uses proxyPort and authless bucket right now.
# Instead of jumping hoops to specify proxyPort in target
# iterator/settings, which only passes down very specific attributes,
# just detect fts instead. The following does not work with
# authless bucket. FTS's worker does its own Couchbase.connect
if not (hasattr(self.ws, "fts") and hasattr(
self.ws.fts, "doc_database_url")):
# default sasl bucket
self.init_db({'bucket': self.ts.bucket, 'host': host, 'port': port,
'username': self.ts.bucket,
'password': self.ts.password})
self.fallingBehindCount = 0
def test_determenistic(self):
docgen = NewNestedDocument(avg_size=self.SIZE)
d1 = docgen.next(key='mykey')
d2 = docgen.next(key='mykey')
d1['body'] = d2['body'] = None
self.assertEqual(d1, d2)
def test_alphabet_size(self):
docgen = NewNestedDocument(avg_size=self.SIZE)
alphabet = docgen._build_alphabet('key')
self.assertEqual(len(alphabet), 64)
