def test_encode_scanvec(self):
# The value is a vbucket's sequence number,
# and guard is a vbucket's UUID.
q = N1QLQuery('SELECT * FROM default')
ms = MutationState()
ms._add_scanvec((42, 3004, 3, 'default'))
q.consistent_with(ms)
dval = json.loads(q.encoded)
sv_exp = {'default': {'42': [3, '3004']}}
self.assertEqual('at_plus', dval['scan_consistency'])
self.assertEqual(sv_exp, dval['scan_vectors'])
# Ensure the vb field gets updated. No duplicates!
ms._add_scanvec((42, 3004, 4, 'default'))
sv_exp['default']['42'] = [4, '3004']
dval = json.loads(q.encoded)
self.assertEqual(sv_exp, dval['scan_vectors'])
ms._add_scanvec((91, 7779, 23, 'default'))
dval = json.loads(q.encoded)
sv_exp['default']['91'] = [23, '7779']
self.assertEqual(sv_exp, dval['scan_vectors'])
# Try with a second bucket
sv_exp['other'] = {'666': [99, '5551212']}
ms._add_scanvec((666, 5551212, 99, 'other'))
dval = json.loads(q.encoded)
self.assertEqual(sv_exp, dval['scan_vectors'])
def test_consistency(self):
uuid = str('10000')
vb = 42
seq = 101
ixname = 'ix'
mutinfo = (vb, uuid, seq, 'dummy-bucket-name')
ms = MutationState()
ms._add_scanvec(mutinfo)
params = cbft.Params()
params.consistent_with(ms)
got = cbft.make_search_body('ix', cbft.MatchNoneQuery(), params)
exp = {
'indexName': ixname,
'query': {
'match_none': None
},
'ctl': {
'consistency': {
'level': 'at_plus',
'vectors': {
ixname: {
'{0}/{1}'.format(vb, uuid): seq
}
}
}
}
}
self.assertEqual(exp, got)
def test_consistency(self):
uuid = str('10000')
vb = 42
seq = 101
ixname = 'ix'
mutinfo = (vb, uuid, seq, 'dummy-bucket-name')
ms = MutationState()
ms._add_scanvec(mutinfo)
params = cbft.Params()
params.consistent_with(ms)
got = cbft.make_search_body('ix', cbft.MatchNoneQuery(), params)
exp = {
'indexName': ixname,
'query': {
'match_none': None
},
'ctl': {
'consistency': {
'level': 'at_plus',
'vectors': {
ixname: {
'{0}/{1}'.format(vb, uuid): seq
}
}
}
}
}
self.assertEqual(exp, got)
def test_encode_scanvec(self):
# The value is a vbucket's sequence number,
# and guard is a vbucket's UUID.
q = N1QLQuery('SELECT * FROM default')
ms = MutationState()
ms._add_scanvec((42, 3004, 3, 'default'))
q.consistent_with(ms)
dval = json.loads(q.encoded)
sv_exp = {
'default': {'42': [3, '3004']}
}
self.assertEqual('at_plus', dval['scan_consistency'])
self.assertEqual(sv_exp, dval['scan_vectors'])
# Ensure the vb field gets updated. No duplicates!
ms._add_scanvec((42, 3004, 4, 'default'))
sv_exp['default']['42'] = [4, '3004']
dval = json.loads(q.encoded)
self.assertEqual(sv_exp, dval['scan_vectors'])
ms._add_scanvec((91, 7779, 23, 'default'))
dval = json.loads(q.encoded)
sv_exp['default']['91'] = [23, '7779']
self.assertEqual(sv_exp, dval['scan_vectors'])
# Try with a second bucket
sv_exp['other'] = {'666': [99, '5551212']}
ms._add_scanvec((666, 5551212, 99, 'other'))
dval = json.loads(q.encoded)
self.assertEqual(sv_exp, dval['scan_vectors'])
def test_mutation_state(self):
cb = self.cb
key = self.gen_key('mutationState')
rv = cb.upsert(key, 'value')
d1 = json.loads(MutationState(rv).encode())
ms = MutationState()
ms.add_results(rv)
d2 = json.loads(ms.encode())
self.assertEqual(d1, d2) # Ensure it's the same
self.assertTrue(d1[cb.bucket]) # Ensure it's not empty
vb, uuid, seq, _ = rv._mutinfo
mt_got = d1[cb.bucket][str(vb)]
self.assertEqual(seq, mt_got[0])
self.assertEqual(str(uuid), mt_got[1])
def test_mutation_state(self):
cb = self.cb
key = self.gen_key('mutationState')
rv = cb.upsert(key, 'value')
d1 = json.loads(MutationState(rv).encode())
ms = MutationState()
ms.add_results(rv)
d2 = json.loads(ms.encode())
self.assertEqual(d1, d2) # Ensure it's the same
self.assertTrue(d1[cb.bucket]) # Ensure it's not empty
vb, uuid, seq, _ = rv._mutinfo
mt_got = d1[cb.bucket][str(vb)]
self.assertEqual(seq, mt_got[0])
self.assertEqual(str(uuid), mt_got[1])
#!/usr/bin/env python
import time
from couchbase.bucket import Bucket
from couchbase.n1ql import N1QLQuery, MutationState
TIMESTAMP = str(time.time())
cb = Bucket('couchbase://localhost/default?fetch_mutation_tokens=true')
rv = cb.upsert('ndoc', {'timestamp': TIMESTAMP})
ms = MutationState()
ms.add_results(rv)
query = N1QLQuery('SELECT * from default WHERE timestamp=$1', TIMESTAMP)
query.consistent_with(ms)
print query.encoded
for row in cb.n1ql_query(query):
print row
def do_batch(self):
if self.ws.n1ql_op == 'read':
curr_items_spot = \
self.curr_items.value - self.ws.creates * self.ws.workers
deleted_spot = \
self.deleted_items.value + self.ws.deletes * self.ws.workers
for _ in xrange(self.BATCH_SIZE):
key = self.existing_keys.next(curr_items_spot, deleted_spot)
doc = self.docs.next(key)
doc['key'] = key
doc['bucket'] = self.ts.bucket
ddoc_name, view_name, query = self.new_queries.next(doc)
self.cb.query(ddoc_name, view_name, query=query)
return
curr_items_tmp = curr_items_spot = self.curr_items.value
if self.ws.n1ql_op == 'create':
with self.lock:
self.curr_items.value += self.BATCH_SIZE
curr_items_tmp = self.curr_items.value - self.BATCH_SIZE
curr_items_spot = (curr_items_tmp -
self.BATCH_SIZE * self.total_workers)
deleted_items_tmp = deleted_spot = 0
if self.ws.n1ql_op == 'delete':
with self.lock:
self.deleted_items.value += self.BATCH_SIZE
deleted_items_tmp = self.deleted_items.value - self.BATCH_SIZE
deleted_spot = (deleted_items_tmp +
self.BATCH_SIZE * self.total_workers)
deleted_capped_items_tmp = deleted_capped_spot = 0
if self.ws.n1ql_op == 'rangedelete':
with self.lock:
self.deleted_capped_items.value += self.BATCH_SIZE
deleted_capped_items_tmp = self.deleted_capped_items.value - self.BATCH_SIZE
deleted_capped_spot = (deleted_capped_items_tmp +
self.BATCH_SIZE * self.total_workers)
casupdated_items_tmp = casupdated_spot = 0
if self.ws.n1ql_op == 'update':
with self.lock:
self.casupdated_items.value += self.BATCH_SIZE
casupdated_items_tmp = self.casupdated_items.value - self.BATCH_SIZE
casupdated_spot = (casupdated_items_tmp +
self.BATCH_SIZE * self.total_workers)
if self.ws.n1ql_op == 'create':
for _ in xrange(self.BATCH_SIZE):
curr_items_tmp += 1
key, ttl = self.new_keys.next(curr_items_tmp)
doc = self.docs.next(key)
doc['key'] = key
doc['bucket'] = self.ts.bucket
ddoc_name, view_name, query = self.new_queries.next(doc)
self.cb.query(ddoc_name, view_name, query=query)
elif self.ws.n1ql_op == 'delete':
for _ in xrange(self.BATCH_SIZE):
deleted_items_tmp += 1
key = self.keys_for_removal.next(deleted_items_tmp)
doc = self.docs.next(key)
doc['key'] = key
doc['bucket'] = self.ts.bucket
ddoc_name, view_name, query = self.new_queries.next(doc)
self.cb.query(ddoc_name, view_name, query=query)
elif self.ws.n1ql_op == 'update' or self.ws.n1ql_op == 'lookupupdate':
for _ in xrange(self.BATCH_SIZE):
key = self.keys_for_casupdate.next(self.sid, curr_items_spot, deleted_spot)
doc = self.docs.next(key)
doc['key'] = key
doc['bucket'] = self.ts.bucket
ddoc_name, view_name, query = self.new_queries.next(doc)
self.cb.query(ddoc_name, view_name, query=query)
elif self.ws.n1ql_op == 'ryow':
for _ in xrange(self.BATCH_SIZE):
query = self.ws.n1ql_queries[0]['statement'][1:-1]
if self.ws.n1ql_queries[0]['prepared'] == "singleton_unique_lookup":
by_key = 'email'
elif self.ws.n1ql_queries[0]['prepared'] == "range_scan":
by_key = 'capped_small'
else:
logger.error('n1ql_queries {} not defined'.format(self.ws.n1ql_queries))
key1 = self.keys_for_casupdate.next(self.sid, curr_items_spot, deleted_spot)
doc1 = self.docs.next(key1)
key2 = self.keys_for_casupdate.next(self.sid, curr_items_spot, deleted_spot)
doc2 = self.docs.next(key2)
rvs = self.cb.client.upsert_multi({key1: doc2, key2: doc1})
# This is a part of requirements:
# Each n1ql worker sleeps for 1 seconds.
time.sleep(float(self.ws.n1ql_queries[0]['time_sleep']))
ms = MutationState()
ms.add_results(*rvs.values())
nq = N1QLQuery(query.format(doc2[by_key]))
nq.consistent_with(ms)
len(list(self.cb.client.n1ql_query(nq)))
elif self.ws.n1ql_op == 'rangeupdate':
for _ in xrange(self.BATCH_SIZE):
key = self.keys_for_casupdate.next(self.sid, curr_items_spot, deleted_spot)
doc = self.docs.next(key)
doc['key'] = key
doc['bucket'] = self.ts.bucket
ddoc_name, view_name, query = self.new_queries.next(doc)
self.cb.query(ddoc_name, view_name, query=query)
elif self.ws.n1ql_op == 'rangedelete':
for _ in xrange(self.BATCH_SIZE):
doc = {}
doc['capped_small'] = "n1ql-_100_" + str(deleted_capped_items_tmp)
ddoc_name, view_name, query = self.new_queries.next(doc)
self.cb.query(ddoc_name, view_name, query=query)
deleted_capped_items_tmp += 1
elif self.ws.n1ql_op == 'merge': #run select * workload for merge
for _ in xrange(self.BATCH_SIZE):
key = self.existing_keys.next(curr_items_spot, deleted_spot)
doc = self.docs.next(key)
doc['key'] = key
doc['bucket'] = self.ts.bucket
ddoc_name, view_name, query = self.new_queries.next(doc)
query['statement'] = "SELECT * FROM `bucket-1` USE KEYS[$1];"
query['args'] = "[\"{key}\"]".format(**doc)
del query['prepared']
self.cb.query(ddoc_name, view_name, query=query)