class ViewWorker(Worker):
NAME = 'query-worker'
def __init__(self, workload_settings, target_settings, shutdown_event):
super().__init__(workload_settings, target_settings, shutdown_event)
self.total_workers = self.ws.query_workers
self.throughput = self.ws.query_throughput
self.reservoir = Reservoir(num_workers=self.ws.query_workers)
if workload_settings.index_type is None:
self.new_queries = ViewQueryGen(workload_settings.ddocs,
workload_settings.query_params)
else:
self.new_queries = ViewQueryGenByType(
workload_settings.index_type, workload_settings.query_params)
@with_sleep
def do_batch(self):
curr_items_spot = \
self.curr_items.value - self.ws.creates * self.ws.workers
deleted_spot = \
self.deleted_items.value + self.ws.deletes * self.ws.workers
for _ in range(self.BATCH_SIZE):
key = self.existing_keys.next(curr_items_spot, deleted_spot)
doc = self.docs.next(key)
ddoc_name, view_name, query = self.new_queries.next(doc)
latency = self.cb.view_query(ddoc_name, view_name, query=query)
self.reservoir.update(operation='query', value=latency)
def run(self, sid, lock, curr_ops, curr_items, deleted_items, *args):
self.cb.start_updater()
if self.throughput < float('inf'):
self.target_time = float(self.BATCH_SIZE) * self.total_workers / \
self.throughput
else:
self.target_time = None
self.sid = sid
self.curr_items = curr_items
self.deleted_items = deleted_items
try:
logger.info('Started: {}-{}'.format(self.NAME, self.sid))
while not self.time_to_stop():
self.do_batch()
except KeyboardInterrupt:
logger.info('Interrupted: {}-{}'.format(self.NAME, self.sid))
else:
logger.info('Finished: {}-{}'.format(self.NAME, self.sid))
self.reservoir.dump(filename='{}-{}'.format(self.NAME, self.sid))
class N1QLWorker(Worker):
NAME = 'query-worker'
def __init__(self,
workload_settings,
target_settings,
shutdown_event=None):
self.new_queries = N1QLQueryGen(workload_settings.n1ql_queries)
self.total_workers = workload_settings.n1ql_workers
self.throughput = workload_settings.n1ql_throughput
self.reservoir = Reservoir(num_workers=workload_settings.n1ql_workers)
super().__init__(workload_settings, target_settings, shutdown_event)
self.init_creds()
def init_keys(self):
self.new_keys = NewOrderedKey(prefix='n1ql', fmtr=self.ws.key_fmtr)
self.existing_keys = UniformKey(prefix='n1ql', fmtr=self.ws.key_fmtr)
self.keys_for_cas_update = KeyForCASUpdate(
total_workers=self.total_workers,
prefix='n1ql',
fmtr=self.ws.key_fmtr)
def init_docs(self):
if self.ws.doc_gen == 'reverse_lookup':
self.docs = ReverseLookupDocument(self.ws.size, prefix='n1ql')
elif self.ws.doc_gen == 'reverse_range_lookup':
self.docs = ReverseRangeLookupDocument(
self.ws.size,
prefix='n1ql',
range_distance=self.ws.range_distance)
elif self.ws.doc_gen == 'ext_reverse_lookup':
self.docs = ExtReverseLookupDocument(self.ws.size,
prefix='n1ql',
num_docs=self.ws.items)
elif self.ws.doc_gen == 'join':
self.docs = JoinedDocument(self.ws.size,
prefix='n1ql',
num_docs=self.ws.items,
num_categories=self.ws.num_categories,
num_replies=self.ws.num_replies)
elif self.ws.doc_gen == 'ref':
self.docs = RefDocument(self.ws.size, prefix='n1ql')
elif self.ws.doc_gen == 'profile':
self.docs = ProfileDocument(self.ws.size, prefix='n1ql')
elif self.ws.doc_gen == 'array_indexing':
self.docs = ArrayIndexingDocument(self.ws.size,
prefix='n1ql',
array_size=self.ws.array_size,
num_docs=self.ws.items)
def init_creds(self):
for bucket in getattr(self.ws, 'buckets', []):
self.cb.client.add_bucket_creds(bucket, self.ts.password)
def read(self):
curr_items_tmp = self.curr_items.value
if self.ws.doc_gen == 'ext_reverse_lookup':
curr_items_tmp //= 4
for _ in range(self.ws.n1ql_batch_size):
key = self.existing_keys.next(curr_items=curr_items_tmp,
curr_deletes=0)
doc = self.docs.next(key)
query = self.new_queries.next(key.string, doc)
latency = self.cb.n1ql_query(query)
self.reservoir.update(operation='query', value=latency)
def create(self):
with self.lock:
self.curr_items.value += self.ws.n1ql_batch_size
curr_items_tmp = self.curr_items.value - self.ws.n1ql_batch_size
for _ in range(self.ws.n1ql_batch_size):
curr_items_tmp += 1
key = self.new_keys.next(curr_items=curr_items_tmp)
doc = self.docs.next(key)
query = self.new_queries.next(key.string, doc)
latency = self.cb.n1ql_query(query)
self.reservoir.update(operation='query', value=latency)
def update(self):
with self.lock:
curr_items_tmp = self.curr_items.value
for _ in range(self.ws.n1ql_batch_size):
key = self.keys_for_cas_update.next(sid=self.sid,
curr_items=curr_items_tmp)
doc = self.docs.next(key)
query = self.new_queries.next(key.string, doc)
latency = self.cb.n1ql_query(query)
self.reservoir.update(operation='query', value=latency)
@with_sleep
def do_batch(self):
if self.ws.n1ql_op == 'read':
self.read()
elif self.ws.n1ql_op == 'create':
self.create()
elif self.ws.n1ql_op == 'update':
self.update()
def run(self, sid, lock, curr_ops, curr_items, *args):
if self.throughput < float('inf'):
self.target_time = self.ws.n1ql_batch_size * self.total_workers / \
float(self.throughput)
else:
self.target_time = None
self.lock = lock
self.sid = sid
self.curr_items = curr_items
try:
logger.info('Started: {}-{}'.format(self.NAME, self.sid))
while not self.time_to_stop():
self.do_batch()
except KeyboardInterrupt:
logger.info('Interrupted: {}-{}'.format(self.NAME, self.sid))
else:
logger.info('Finished: {}-{}'.format(self.NAME, self.sid))
self.reservoir.dump(filename='{}-{}'.format(self.NAME, self.sid))
class KVWorker(Worker):
NAME = 'kv-worker'
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.reservoir = Reservoir(num_workers=self.ws.workers)
def gen_cmd_sequence(self, cb=None, extras=None) -> Sequence:
ops = \
['c'] * self.ws.creates + \
['r'] * self.ws.reads + \
['u'] * self.ws.updates + \
['d'] * self.ws.deletes + \
['ru'] * (self.ws.reads_and_updates // 2) + \
['fus'] * self.ws.fts_updates_swap + \
['fur'] * self.ws.fts_updates_reverse
random.shuffle(ops)
curr_items_tmp = curr_items_spot = self.curr_items.value
if self.ws.creates:
with self.lock:
self.curr_items.value += self.ws.creates
curr_items_tmp = self.curr_items.value - self.ws.creates
curr_items_spot = (curr_items_tmp -
self.ws.creates * self.ws.workers)
deleted_items_tmp = deleted_spot = 0
if self.ws.deletes:
with self.lock:
self.deleted_items.value += self.ws.deletes
deleted_items_tmp = self.deleted_items.value - self.ws.deletes
deleted_spot = (deleted_items_tmp +
self.ws.deletes * self.ws.workers)
if not cb:
cb = self.cb
cmds = []
for op in ops:
if op == 'c':
key = self.new_keys.next(curr_items_tmp)
doc = self.docs.next(key)
curr_items_tmp += 1
cmds.append((None, cb.create, (key.string, doc)))
elif op == 'r':
key = self.existing_keys.next(curr_items_spot, deleted_spot)
if extras == 'subdoc':
cmds.append(
('get', cb.read, (key.string, self.ws.subdoc_field)))
elif extras == 'xattr':
cmds.append(('get', cb.read_xattr, (key.string,
self.ws.xattr_field)))
else:
cmds.append(('get', cb.read, (key.string, )))
elif op == 'u':
key = self.existing_keys.next(curr_items_spot, deleted_spot,
self.current_hot_load_start,
self.timer_elapse)
doc = self.docs.next(key)
if extras == 'subdoc':
cmds.append(('set', cb.update,
(key.string, self.ws.subdoc_field, doc)))
elif extras == 'xattr':
cmds.append(('set', cb.update_xattr,
(key.string, self.ws.xattr_field, doc)))
else:
cmds.append(('set', cb.update, (key.string, doc)))
elif op == 'd':
key = self.keys_for_removal.next(deleted_items_tmp)
deleted_items_tmp += 1
cmds.append((None, cb.delete, (key.string, )))
elif op == 'ru':
key = self.existing_keys.next(curr_items_spot, deleted_spot)
doc = self.docs.next(key)
cmds.append(('get', cb.read, (key.string, )))
cmds.append(('set', cb.update, (key.string, doc)))
elif op == 'fus':
key = self.fts_keys.next()
cmds.append((None, self.do_fts_updates_swap, (key, )))
elif op == 'fur':
key = self.fts_keys.next()
cmds.append((None, self.do_fts_updates_reverse, (key, )))
return cmds
def do_fts_updates_swap(self, key):
doc = self.cb.client.get(key).value
if 'text' in doc and 'text2' in doc:
tmp = doc["text2"]
doc["text2"] = doc["text"]
doc["text"] = tmp
elif 'time' in doc:
if randint(0, 1):
doc["time"] = int(doc["time"]) >> 1
else:
doc["time"] = int(doc["time"]) << 1
else:
return
self.cb.client.set(key, doc)
def do_fts_updates_reverse(self, key):
doc = self.cb.client.get(key).value
words = doc["name"].split(' ')
if len(words):
doc["name"] = ' '.join(words[::-1])
self.cb.client.set(key, doc)
@with_sleep
def do_batch(self, *args, **kwargs):
for cmd, func, args in self.gen_cmd_sequence():
latency = func(*args)
if latency is not None:
self.reservoir.update(operation=cmd, value=latency)
def run_condition(self, curr_ops):
return curr_ops.value < self.ws.ops and not self.time_to_stop()
def run(self,
sid,
lock,
curr_ops,
curr_items,
deleted_items,
current_hot_load_start=None,
timer_elapse=None):
if self.ws.throughput < float('inf'):
self.target_time = float(self.BATCH_SIZE) * self.ws.workers / \
self.ws.throughput
else:
self.target_time = None
self.sid = sid
self.lock = lock
self.curr_items = curr_items
self.deleted_items = deleted_items
self.current_hot_load_start = current_hot_load_start
self.timer_elapse = timer_elapse
self.seed()
logger.info('Started: {}-{}'.format(self.NAME, self.sid))
try:
while self.run_condition(curr_ops):
with lock:
curr_ops.value += self.BATCH_SIZE
self.do_batch()
self.report_progress(curr_ops.value)
except KeyboardInterrupt:
logger.info('Interrupted: {}-{}'.format(self.NAME, self.sid))
else:
logger.info('Finished: {}-{}'.format(self.NAME, self.sid))
self.reservoir.dump(filename='{}-{}'.format(self.NAME, self.sid))
class N1QLWorker(Worker):
NAME = 'n1ql-worker'
def __init__(self, workload_settings, target_settings, shutdown_event):
self.new_queries = N1QLQueryGen(workload_settings.n1ql_queries)
self.total_workers = workload_settings.n1ql_workers
self.throughput = workload_settings.n1ql_throughput
self.reservoir = Reservoir(num_workers=workload_settings.n1ql_workers)
super(N1QLWorker, self).__init__(workload_settings, target_settings,
shutdown_event)
def init_keys(self):
self.existing_keys = ExistingKey(self.ws.working_set,
self.ws.working_set_access,
prefix='n1ql')
self.new_keys = NewKey(prefix='n1ql', expiration=self.ws.expiration)
self.keys_for_casupdate = KeyForCASUpdate(self.total_workers,
self.ws.working_set,
self.ws.working_set_access,
prefix='n1ql')
def init_docs(self):
if self.ws.doc_gen == 'reverse_lookup':
self.docs = ReverseLookupDocument(self.ws.size,
prefix='n1ql')
elif self.ws.doc_gen == 'reverse_range_lookup':
self.docs = ReverseRangeLookupDocument(self.ws.size,
prefix='n1ql',
range_distance=self.ws.range_distance)
elif self.ws.doc_gen == 'ext_reverse_lookup':
self.docs = ExtReverseLookupDocument(self.ws.size,
prefix='n1ql',
num_docs=self.ws.items)
elif self.ws.doc_gen == 'join':
self.docs = JoinedDocument(self.ws.size,
prefix='n1ql',
num_docs=self.ws.items,
num_categories=self.ws.num_categories,
num_replies=self.ws.num_replies)
elif self.ws.doc_gen == 'ref':
self.docs = RefDocument(self.ws.size,
prefix='n1ql')
elif self.ws.doc_gen == 'array_indexing':
self.docs = ArrayIndexingDocument(self.ws.size,
prefix='n1ql',
array_size=self.ws.array_size,
num_docs=self.ws.items)
def init_db(self):
host, port = self.ts.node.split(':')
self.cb = N1QLGen(bucket=self.ts.bucket, password=self.ts.password,
host=host, port=port)
def read(self):
curr_items_tmp = self.curr_items.value
if self.ws.doc_gen == 'ext_reverse_lookup':
curr_items_tmp /= 4
for _ in range(self.BATCH_SIZE):
key = self.existing_keys.next(curr_items=curr_items_tmp,
curr_deletes=0)
doc = self.docs.next(key)
doc['key'] = key
doc['bucket'] = self.ts.bucket
query = self.new_queries.next(doc)
_, latency = self.cb.query(query)
self.reservoir.update(latency)
def create(self):
with self.lock:
self.curr_items.value += self.BATCH_SIZE
curr_items_tmp = self.curr_items.value - self.BATCH_SIZE
for _ in range(self.BATCH_SIZE):
curr_items_tmp += 1
key, ttl = self.new_keys.next(curr_items=curr_items_tmp)
doc = self.docs.next(key)
doc['key'] = key
doc['bucket'] = self.ts.bucket
query = self.new_queries.next(doc)
_, latency = self.cb.query(query)
self.reservoir.update(latency)
def update(self):
with self.lock:
self.cas_updated_items.value += self.BATCH_SIZE
curr_items_tmp = self.curr_items.value - self.BATCH_SIZE
for _ in range(self.BATCH_SIZE):
key = self.keys_for_casupdate.next(self.sid,
curr_items=curr_items_tmp,
curr_deletes=0)
doc = self.docs.next(key)
doc['key'] = key
doc['bucket'] = self.ts.bucket
query = self.new_queries.next(doc)
_, latency = self.cb.query(query)
self.reservoir.update(latency)
def range_update(self):
with self.lock:
self.cas_updated_items.value += self.BATCH_SIZE
curr_items_tmp = self.curr_items.value - self.BATCH_SIZE
for _ in range(self.BATCH_SIZE):
key = self.keys_for_casupdate.next(self.sid,
curr_items=curr_items_tmp,
curr_deletes=0)
doc = self.docs.next(key)
doc['key'] = key
doc['bucket'] = self.ts.bucket
query = self.new_queries.next(doc)
_, latency = self.cb.query(query)
self.reservoir.update(latency)
@with_sleep
def do_batch(self):
if self.ws.n1ql_op == 'read':
self.read()
elif self.ws.n1ql_op == 'create':
self.create()
elif self.ws.n1ql_op == 'update':
self.update()
elif self.ws.n1ql_op == 'rangeupdate':
self.range_update()
def run(self, sid, lock, curr_queries, curr_items, deleted_items,
cas_updated_items):
if self.throughput < float('inf'):
self.target_time = float(self.BATCH_SIZE) * self.total_workers / \
self.throughput
else:
self.target_time = None
self.lock = lock
self.sid = sid
self.curr_items = curr_items
self.cas_updated_items = cas_updated_items
self.curr_queries = curr_queries
try:
logger.info('Started: {}-{}'.format(self.NAME, self.sid))
while curr_queries.value < self.ws.ops and not self.time_to_stop():
with self.lock:
curr_queries.value += self.BATCH_SIZE
self.do_batch()
except (KeyboardInterrupt, ValueFormatError, AttributeError) as e:
logger.info('Interrupted: {}-{}-{}'.format(self.NAME, self.sid, e))
else:
logger.info('Finished: {}-{}'.format(self.NAME, self.sid))
self.reservoir.dump(filename='{}-{}'.format(self.NAME, self.sid))