class SDKClient(threading.Thread):
def __init__(self, name, task, e):
threading.Thread.__init__(self)
self.name = name
self.i = 0
self.op_factor = CLIENTSPERPROCESS * PROCSPERTASK
self.ops_sec = task['ops_sec']
self.bucket = task['bucket']
self.password = task['password']
self.template = task['template']
self.create_count = task['create_count']/self.op_factor
self.update_count = task['update_count']/self.op_factor
self.get_count = task['get_count']/self.op_factor
self.del_count = task['del_count']/self.op_factor
self.exp_count = task['exp_count']/self.op_factor
self.consume_queue = task['consume_queue']
self.ttl = task['ttl']
self.miss_perc = task['miss_perc']
self.active_hosts = task['active_hosts']
self.batch_size = 5000
self.memq = queue.Queue()
self.hotset = []
self.ccq = None
self.hotkeys = []
if task['template']['cc_queues']:
self.ccq = str(task['template']['cc_queues'][0]) #only supporting 1 now
RabbitHelper().declare(self.ccq)
self.batch_size = 1000
if self.batch_size > self.create_count:
self.batch_size = self.create_count
self.active_hosts = task['active_hosts']
if not self.active_hosts:
self.active_hosts = [cfg.COUCHBASE_IP]
addr = task['active_hosts'][random.randint(0,len(self.active_hosts) - 1)].split(':')
host = addr[0]
port = 8091
if len(addr) > 1:
port = addr[1]
self.cb = GConnection(bucket=self.bucket, password = self.password, host = host, port = port)
self.e = e
def run(self):
cycle = ops_total = 0
while self.e.is_set() == False:
start = datetime.datetime.now()
# do an op cycle
threads = self.do_cycle()
gevent.joinall(threads)
# wait till next cycle
end = datetime.datetime.now()
wait = 1 - (end - start).microseconds/float(1000000)
if (wait > 0):
time.sleep(wait)
else:
pass #probably we are overcomitted, but it's ok
ops_total = ops_total + self.ops_sec
cycle = cycle + 1
if (cycle % 100) == 0:
logging.info("[Thread %s] total ops: %s" % (self.name, ops_total))
if self.memq.qsize() > 100:
self.flushq()
# push everything to rabbitmq
self.flushq()
def flushq(self):
if self.ccq is not None:
# declare queue
mq = RabbitHelper()
mq.declare(self.ccq)
while self.memq.empty() == False:
try:
msg = self.memq.get_nowait()
msg = json.dumps(msg)
mq.putMsg(self.ccq, msg)
except queue.Empty:
pass
# hot keys
if len(self.hotkeys) > 0:
key_map = {'start' : self.hotkeys[0],
'end' : self.hotkeys[-1]}
msg = json.dumps(key_map)
mq.putMsg(self.ccq, msg)
def do_cycle(self):
threads = []
if self.create_count > 0:
count = self.create_count
docs_to_expire = self.exp_count
# check if we need to expire some docs
if docs_to_expire > 0:
# create an expire batch
self.mset(self.template['kv'], docs_to_expire, ttl = self.ttl)
count = count - docs_to_expire
t = gevent.spawn(self.mset, self.template['kv'], count)
threads.append(t)
if self.update_count > 0:
t = gevent.spawn(self.mset_update, self.template['kv'], self.update_count)
threads.append(t)
if self.get_count > 0:
t = gevent.spawn(self.mget, self.get_count)
threads.append(t)
if self.del_count > 0:
t = gevent.spawn(self.mdelete, self.del_count)
threads.append(t)
return threads
def mset(self, template, count, ttl = 0):
msg = {}
keys = []
cursor = 0
j = 0
for j in xrange(count):
self.i = self.i+1
msg[self.name+str(self.i)] = template
keys.append(self.name+str(self.i))
if ((j+1) % self.batch_size) == 0:
batch = keys[cursor:j+1]
self.memq.put_nowait({'start' : batch[0],
'end' : batch[-1]})
self._mset(msg, ttl)
cursor = j + 1
msg = {}
if (cursor < j) and (len(msg) > 0):
self._mset(msg, ttl)
self.memq.put_nowait({'start' : keys[cursor],
'end' : keys[-1]})
def _mset(self, msg, ttl = 0):
try:
self.cb.set_multi(msg, ttl=ttl)
except TemporaryFailError:
logging.warn("temp failure during mset - cluster may be unstable")
except TimeoutError:
logging.warn("cluster timed trying to handle mset")
def mset_update(self, template, count):
msg = {}
batches = self.getKeys(count)
if len(batches) > 0:
for batch in batches:
try:
for key in batch:
msg[key] = template
self.cb.set_multi(msg)
except NotFoundError as nf:
logging.error("update key not found! %s: " % nf.key)
except TimeoutError:
logging.warn("cluster timed out trying to handle mset - cluster may be unstable")
except TemporaryFailError:
logging.warn("temp failure during mset - cluster may be unstable")
def mget(self, count):
batches = []
if self.miss_perc > 0:
batches = self.getCacheMissKeys(count)
else:
batches = self.getKeys(count)
if len(batches) > 0:
for batch in batches:
try:
self.cb.get_multi(batch)
except NotFoundError as nf:
logging.warn("get key not found! %s: " % nf.key)
except TimeoutError:
logging.warn("cluster timed out trying to handle mget - cluster may be unstable")
def mdelete(self, count):
batches = self.getKeys(count, requeue = False)
keys_deleted = 0
# delete from buffer
if len(batches) > 0:
keys_deleted = self._mdelete(batches)
else:
pass
def _mdelete(self, batches):
keys_deleted = 0
for batch in batches:
try:
if len(batch) > 0:
keys_deleted = len(batch) + keys_deleted
self.cb.delete_multi(batch)
except NotFoundError as nf:
logging.warn("get key not found! %s: " % nf.key)
except TimeoutError:
logging.warn("cluster timed out trying to handle mdelete - cluster may be unstable")
return keys_deleted
def getCacheMissKeys(self, count):
# returns batches of keys where first batch contains # of keys to miss
keys_retrieved = 0
batches = []
miss_keys = []
requeue = len(self.hotkeys) > 0
keys = self.getKeysFromQueue(requeue = requeue, force_stale = True)
if requeue == False:
# hotkeys were taken off queue and cannot be reused
self.hotkeys = keys
if len(keys) > 0:
# miss% of count keys
num_to_miss = int( ((self.miss_perc/float(100)) * len(keys)) / float(self.op_factor))
miss_keys = keys[:num_to_miss]
batches.append(miss_keys)
keys_retrieved = len(miss_keys)
# use old hotkeys for rest of set
while keys_retrieved < count:
keys = self.hotkeys
# in case we got too many keys slice the batch
need = count - keys_retrieved
if(len(keys) > need):
keys = keys[:need]
keys_retrieved = keys_retrieved + len(keys)
# add to batch
batches.append(keys)
return batches
def getKeys(self, count, requeue = True):
keys_retrieved = 0
batches = []
while keys_retrieved < count:
# get keys
keys = self.getKeysFromQueue(requeue)
if len(keys) == 0:
break
# in case we got too many keys slice the batch
need = count - keys_retrieved
if(len(keys) > need):
keys = keys[:need]
keys_retrieved = keys_retrieved + len(keys)
# add to batch
batches.append(keys)
return batches
def getKeysFromQueue(self, requeue = True, force_stale = False):
# get key mapping and convert to keys
keys = []
key_map = None
# priority to stale queue
if force_stale:
key_map = self.getKeyMapFromRemoteQueue(requeue)
# fall back to local qeueue
if key_map is None:
key_map = self.getKeyMapFromLocalQueue(requeue)
if key_map:
keys = self.keyMapToKeys(key_map)
return keys
def keyMapToKeys(self, key_map):
keys = []
# reconstruct key-space
prefix, start_idx = key_map['start'].split('_')
prefix, end_idx = key_map['end'].split('_')
for i in range(int(start_idx), int(end_idx) + 1):
keys.append(prefix+"_"+str(i))
return keys
def fillq(self):
if self.ccq == None:
return
# put about 20 items into the queue
for i in xrange(20):
key_map = self.getKeyMapFromRemoteQueue()
if key_map:
self.memq.put_nowait(key_map)
def getKeyMapFromLocalQueue(self, requeue = True):
key_map = None
try:
key_map = self.memq.get_nowait()
if requeue:
self.memq.put_nowait(key_map)
except queue.Empty:
#no more items
if self.ccq is not None:
self.fillq()
return key_map
def getKeyMapFromRemoteQueue(self, requeue = True):
key_map = None
mq = RabbitHelper()
if mq.qsize(self.ccq) > 0:
try:
key_map = mq.getJsonMsg(self.ccq, requeue = requeue )
except Exception:
pass
return key_map
class SDKClient(threading.Thread):
def __init__(self, name, task, e):
threading.Thread.__init__(self)
self.name = name
self.i = 0
self.op_factor = CLIENTSPERPROCESS * PROCSPERTASK
self.ops_sec = task['ops_sec']
self.bucket = task['bucket']
self.password = task['password']
self.template = task['template']
self.default_tsizes = [128, 256]
self.create_count = task['create_count']/self.op_factor
self.update_count = task['update_count']/self.op_factor
self.get_count = task['get_count']/self.op_factor
self.del_count = task['del_count']/self.op_factor
self.exp_count = task['exp_count']/self.op_factor
self.ttl = task['ttl']
self.miss_perc = task['miss_perc']
self.active_hosts = task['active_hosts']
self.batch_size = 5000
self.memq = queue.Queue()
self.consume_queue = task['consume_queue']
self.standalone = task['standalone']
self.ccq = None
self.hotkey_batches = []
if self.consume_queue is not None:
RabbitHelper().declare(self.consume_queue)
if task['template']['cc_queues']:
self.ccq = str(task['template']['cc_queues'][0]) #only supporting 1 now
RabbitHelper().declare(self.ccq)
if self.batch_size > self.create_count:
self.batch_size = self.create_count
self.active_hosts = task['active_hosts']
if not self.active_hosts:
self.active_hosts = [cfg.COUCHBASE_IP]
addr = task['active_hosts'][random.randint(0,len(self.active_hosts) - 1)].split(':')
host = addr[0]
port = 8091
if len(addr) > 1:
port = addr[1]
self.e = e
self.cb = None
self.isterminal = False
self.done = False
try:
self.cb = GConnection(bucket=self.bucket, password = self.password, host = host, port = port)
except Exception as ex:
logging.error("[Thread %s] cannot reach %s:%s/%s" %
(self.name, host, port, self.bucket))
logging.error(ex)
self.isterminal = True
logging.info("[Thread %s] started for workload: %s" % (self.name, task['id']))
def run(self):
cycle = ops_total = 0
self.e.set()
while self.e.is_set() == True:
start = datetime.datetime.now()
# do an op cycle
self.do_cycle()
if self.isterminal == True:
# some error occured during workload
self.flushq(True)
exit(-1)
# wait till next cycle
end = datetime.datetime.now()
wait = 1 - (end - start).microseconds/float(1000000)
if (wait > 0):
time.sleep(wait)
else:
pass #probably we are overcomitted, but it's ok
ops_total = ops_total + self.ops_sec
cycle = cycle + 1
if (cycle % 120) == 0: # 2 mins
logging.info("[Thread %s] total ops: %s" % (self.name, ops_total))
self.flushq()
self.flushq()
logging.info("[Thread %s] done!" % (self.name))
def flushq(self, flush_hotkeys = False):
if self.standalone:
return
mq = RabbitHelper()
if self.ccq is not None:
logging.info("[Thread %s] flushing %s items to %s" %
(self.name, self.memq.qsize(), self.ccq))
# declare queue
mq.declare(self.ccq)
# empty the in memory queue
while self.memq.empty() == False:
try:
msg = self.memq.get_nowait()
msg = json.dumps(msg)
mq.putMsg(self.ccq, msg)
except queue.Empty:
pass
# hot keys
if flush_hotkeys and (len(self.hotkey_batches) > 0):
# try to put onto remote queue
queue = self.consume_queue or self.ccq
if queue is not None:
key_map = {'start' : self.hotkey_batches[0][0],
'end' : self.hotkey_batches[-1][-1]}
msg = json.dumps(key_map)
mq.putMsg(queue, msg)
self.hotkey_batches = []
def do_cycle(self):
sizes = self.template.get('size') or self.default_tsizes
t_size = sizes[random.randint(0,len(sizes)-1)]
self.template['t_size'] = t_size
if self.create_count > 0:
count = self.create_count
docs_to_expire = self.exp_count
# check if we need to expire some docs
if docs_to_expire > 0:
# create an expire batch
self.mset(self.template, docs_to_expire, ttl = self.ttl)
count = count - docs_to_expire
self.mset(self.template, count)
if self.update_count > 0:
self.mset_update(self.template, self.update_count)
if self.get_count > 0:
self.mget(self.get_count)
if self.del_count > 0:
self.mdelete(self.del_count)
def mset(self, template, count, ttl = 0):
msg = {}
keys = []
cursor = 0
j = 0
template = resolveTemplate(template)
for j in xrange(count):
self.i = self.i+1
msg[self.name+str(self.i)] = template
keys.append(self.name+str(self.i))
if ((j+1) % self.batch_size) == 0:
batch = keys[cursor:j+1]
self._mset(msg, ttl)
self.memq.put_nowait({'start' : batch[0],
'end' : batch[-1]})
msg = {}
cursor = j
elif j == (count -1):
batch = keys[cursor:]
self._mset(msg, ttl)
self.memq.put_nowait({'start' : batch[0],
'end' : batch[-1]})
def _mset(self, msg, ttl = 0):
try:
self.cb.set_multi(msg, ttl=ttl)
except TemporaryFailError:
logging.warn("temp failure during mset - cluster may be unstable")
except TimeoutError:
logging.warn("cluster timed trying to handle mset")
except NetworkError as nx:
logging.error("network error")
logging.error(nx)
except Exception as ex:
logging.error(ex)
self.isterminal = True
def mset_update(self, template, count):
msg = {}
batches = self.getKeys(count)
template = resolveTemplate(template)
if len(batches) > 0:
for batch in batches:
try:
for key in batch:
msg[key] = template
self.cb.set_multi(msg)
except NotFoundError as nf:
logging.error("update key not found! %s: " % nf.key)
except TimeoutError:
logging.warn("cluster timed out trying to handle mset - cluster may be unstable")
except NetworkError as nx:
logging.error("network error")
logging.error(nx)
except TemporaryFailError:
logging.warn("temp failure during mset - cluster may be unstable")
except Exception as ex:
logging.error(ex)
self.isterminal = True
def mget(self, count):
batches = []
if self.miss_perc > 0:
batches = self.getCacheMissKeys(count)
else:
batches = self.getKeys(count)
if len(batches) > 0:
for batch in batches:
try:
self.cb.get_multi(batch)
except NotFoundError as nf:
logging.warn("get key not found! %s: " % nf.key)
pass
except TimeoutError:
logging.warn("cluster timed out trying to handle mget - cluster may be unstable")
except NetworkError as nx:
logging.error("network error")
logging.error(nx)
except Exception as ex:
logging.error(ex)
self.isterminal = True
def mdelete(self, count):
batches = self.getKeys(count, requeue = False)
keys_deleted = 0
# delete from buffer
if len(batches) > 0:
keys_deleted = self._mdelete(batches)
else:
pass
def _mdelete(self, batches):
keys_deleted = 0
for batch in batches:
try:
if len(batch) > 0:
keys_deleted = len(batch) + keys_deleted
self.cb.delete_multi(batch)
except NotFoundError as nf:
logging.warn("get key not found! %s: " % nf.key)
except TimeoutError:
logging.warn("cluster timed out trying to handle mdelete - cluster may be unstable")
except NetworkError as nx:
logging.error("network error")
logging.error(nx)
except Exception as ex:
logging.error(ex)
self.isterminal = True
return keys_deleted
def getCacheMissKeys(self, count):
# returns batches of keys where first batch contains # of keys to miss
keys_retrieved = 0
batches = []
miss_keys = []
num_to_miss = int( ((self.miss_perc/float(100)) * count))
miss_batches = self.getKeys(num_to_miss, force_stale = True)
if len(self.hotkey_batches) == 0:
# hotkeys are taken off queue and cannot be reused
# until workload is flushed
need = count - num_to_miss
self.hotkey_batches = self.getKeys(need, requeue = False)
batches = miss_batches + self.hotkey_batches
return batches
def getKeys(self, count, requeue = True, force_stale = False):
keys_retrieved = 0
batches = []
while keys_retrieved < count:
# get keys
keys = self.getKeysFromQueue(requeue, force_stale = force_stale)
if len(keys) == 0:
break
# in case we got too many keys slice the batch
need = count - keys_retrieved
if(len(keys) > need):
keys = keys[:need]
keys_retrieved = keys_retrieved + len(keys)
# add to batch
batches.append(keys)
return batches
def getKeysFromQueue(self, requeue = True, force_stale = False):
# get key mapping and convert to keys
keys = []
key_map = None
# priority to stale queue
if force_stale:
key_map = self.getKeyMapFromRemoteQueue(requeue)
# fall back to local qeueue
if key_map is None:
key_map = self.getKeyMapFromLocalQueue(requeue)
if key_map:
keys = self.keyMapToKeys(key_map)
return keys
def keyMapToKeys(self, key_map):
keys = []
# reconstruct key-space
prefix, start_idx = key_map['start'].split('_')
prefix, end_idx = key_map['end'].split('_')
for i in range(int(start_idx), int(end_idx) + 1):
keys.append(prefix+"_"+str(i))
return keys
def fillq(self):
if (self.consume_queue == None) and (self.ccq == None):
return
# put about 20 items into the queue
for i in xrange(20):
key_map = self.getKeyMapFromRemoteQueue()
if key_map:
self.memq.put_nowait(key_map)
logging.info("[Thread %s] filled %s items from %s" %
(self.name, self.memq.qsize(), self.consume_queue or self.ccq))
def getKeyMapFromLocalQueue(self, requeue = True):
key_map = None
try:
key_map = self.memq.get_nowait()
if requeue:
self.memq.put_nowait(key_map)
except queue.Empty:
#no more items
self.fillq()
return key_map
def getKeyMapFromRemoteQueue(self, requeue = True):
key_map = None
mq = RabbitHelper()
# try to fetch from consume queue and
# fall back to ccqueue
queue = self.consume_queue
if queue is None or mq.qsize(queue) == 0:
queue = self.ccq
if mq.qsize(queue) > 0:
try:
key_map = mq.getJsonMsg(queue, requeue = requeue )
except Exception:
pass
return key_map