def worker_init():
# cleanup queues
rabbitHelper = RabbitHelper()
cached_queues = WorkloadCacher().queues + TemplateCacher().cc_queues
test_queues = ["workload","workload_template", "admin_tasks", "xdcr_tasks"] + cached_queues
for queue in test_queues:
try:
if rabbitHelper.qsize(queue) > 0:
print "Purge Queue: "+queue +" "+ str(rabbitHelper.qsize(queue))
rabbitHelper.purge(queue)
except Exception as ex:
print ex
cacheClean()
# kill old background processes
kill_procs=["sdkserver"]
for proc in kill_procs:
os.system("ps aux | grep %s | awk '{print $2}' | xargs kill" % proc)
# start sdk servers
os.system("ruby sdkserver.rb &")
os.system("python sdkserver.py &")
# make sure logdir exists
os.system("mkdir -p "+cfg.LOGDIR)
def get_remote_phase_status(remoteIP, taskID, retry=5):
# assemble a request to remoteIP phase_status method
rabbitHelper = RabbitHelper(mq_server=remoteIP)
rcq = getResponseQueue(rabbitHelper)
task_method = "app.systest_manager.get_phase_status"
task_queue = "phase_status_" + cfg.CB_CLUSTER_TAG
task_routing_queue = cfg.CB_CLUSTER_TAG + ".phase.status"
args = (taskID, rcq)
# call phase_status task
rawTaskPublisher(task_method,
args,
task_queue,
broker=remoteIP,
exchange="default",
routing_key=task_routing_queue)
# retrieve status of phase
rc = None
while rc is None and retry > 0:
rc = rabbitHelper.getMsg(rcq)
time.sleep(2)
retry = retry - 1
rabbitHelper.delete(rcq)
return rc == 'True'
def import_template(args):
val = None
if args.type == "json":
json_val = {}
for kv in args.kvpairs:
pair = '{%s}' % kv
try:
pair = json.loads(pair)
json_val.update(pair)
except ValueError as ex:
print("ERROR: Unable to encode as valid json: %s " % kv)
print("make sure strings surrounded by double quotes")
return
val = json_val
#TODO binary blobs
template = {
"name": args.name,
"ttl": args.ttl,
"flags": args.flags,
"cc_queues": args.cc_queues,
"size": args.size,
"kv": val
}
cluster = args.cluster
rabbitHelper = RabbitHelper(args.broker, cluster)
template['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg("workload_template_" + cluster, json.dumps(template))
receiveResponse(rabbitHelper, template['rcq'])
def run_workload(args):
workload = {}
if args.name != None:
# TODO: read in workload params from saved store
# workload.update(cached_workload)
pass
if args.wait is not None:
args.wait = conv_to_secs(args.wait)
workload = { "bucket" : args.bucket,
"password" : args.password,
"ops_per_sec" : args.ops,
"create_perc" : args.create,
"update_perc" : args.update,
"get_perc" : args.get,
"del_perc" : args.delete,
"exp_perc" : args.expire,
"miss_perc" : args.miss,
"ttl" : args.ttl,
"cc_queues" : args.cc_queues,
"consume_queue" : args.consume_queue,
"postconditions" : args.postcondition,
"preconditions" : args.precondition,
"wait" : args.wait,
"template" : args.template}
cluster = args.cluster
rabbitHelper = RabbitHelper(args.broker, cluster)
workload['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg("workload_"+cluster, json.dumps(workload))
receiveResponse(rabbitHelper, workload['rcq'])
def get_remote_phase_status(remoteIP, taskID, retry = 5):
# assemble a request to remoteIP phase_status method
rabbitHelper = RabbitHelper(mq_server = remoteIP)
rcq = getResponseQueue(rabbitHelper)
task_method = "app.systest_manager.get_phase_status"
task_queue = "phase_status_"+cfg.CB_CLUSTER_TAG
task_routing_queue = cfg.CB_CLUSTER_TAG+".phase.status"
args = (taskID, rcq)
# call phase_status task
rawTaskPublisher(task_method,
args,
task_queue,
broker = remoteIP,
exchange="default",
routing_key = task_routing_queue)
# retrieve status of phase
rc = None
while rc is None and retry > 0:
rc = rabbitHelper.getMsg(rcq)
time.sleep(2)
retry = retry - 1
rabbitHelper.delete(rcq)
return rc == 'True'
def run_workload(args):
workload = {}
if args.name != None:
# TODO: read in workload params from saved store
# workload.update(cached_workload)
pass
if args.wait is not None:
args.wait = conv_to_secs(args.wait)
if args.expires is not None:
args.expires = conv_to_secs(args.expires)
workload = {
"bucket": args.bucket,
"ops_per_sec": args.ops,
"create_perc": args.create,
"update_perc": args.update,
"get_perc": args.get,
"del_perc": args.delete,
"cc_queues": args.cc_queues,
"consume_queue": args.consume_queue,
"postconditions": args.postcondition,
"preconditions": args.precondition,
"wait": args.wait,
"expires": args.expires,
"template": args.template
}
rabbitHelper = RabbitHelper(args.broker)
rabbitHelper.putMsg("workload", json.dumps(workload))
def run_workload(args):
workload = {}
if args.name != None:
# TODO: read in workload params from saved store
# workload.update(cached_workload)
pass
if args.wait is not None:
args.wait = conv_to_secs(args.wait)
if args.expires is not None:
args.expires = conv_to_secs(args.expires)
workload = { "bucket" : args.bucket,
"ops_per_sec" : args.ops,
"create_perc" : args.create,
"update_perc" : args.update,
"get_perc" : args.get,
"del_perc" : args.delete,
"cc_queues" : args.cc_queues,
"consume_queue" : args.consume_queue,
"postconditions" : args.postcondition,
"preconditions" : args.precondition,
"wait" : args.wait,
"expires" : args.expires,
"template" : args.template}
rabbitHelper = RabbitHelper(args.broker)
rabbitHelper.putMsg("workload", json.dumps(workload))
def requeueNonDeletedKeys(self):
rabbitHelper = RabbitHelper()
task_type = 'app.sdk_client_tasks.mdelete'
# requeue pending delete keys so that they may be deleted in another workload
while rabbitHelper.qsize(self.task_queue) > 0:
task_set = rabbitHelper.getJsonMsg(self.task_queue)
if len(task_set) > 0:
keys = [task['args'] for task in task_set \
if task['task'] == task_type]
if len(keys) > 0:
# put back on to consume_queue
msg = json.dumps(keys[0][0])
rabbitHelper.putMsg(self.consume_queue, msg)
try:
# delete task queue
rabbitHelper.delete(self.task_queue)
# delete consume queue if it was a miss_queue
if self.miss_queue is not None and self.consume_queue is not None:
rabbitHelper.delete(self.consume_queue)
except:
pass
def runRemotePhases(remote_phases, retry=5):
taskIds = []
for remoteIP in remote_phases:
# get handler to remote broker
rabbitHelper = RabbitHelper(mq_server=remoteIP)
rcq = getResponseQueue(rabbitHelper)
args = (remote_phases[remoteIP], rcq)
# call runPhase on remoteIP
rawTaskPublisher("app.systest_manager.runPhase",
args,
"run_phase_" + cfg.CB_CLUSTER_TAG,
broker=remoteIP,
exchange="default",
routing_key=cfg.CB_CLUSTER_TAG + ".run.phase")
# get taskID of phase running on remote broker
taskId = None
while taskId is None and retry > 0:
time.sleep(2)
taskId = rabbitHelper.getMsg(rcq)
taskIds.append((remoteIP, taskId))
retry = retry - 1
return taskIds
def runRemotePhases(remote_phases, retry = 5):
taskIds = []
for remoteIP in remote_phases:
# get handler to remote broker
rabbitHelper = RabbitHelper(mq_server = remoteIP)
rcq = getResponseQueue(rabbitHelper)
args = (remote_phases[remoteIP], rcq)
# call runPhase on remoteIP
rawTaskPublisher("app.systest_manager.runPhase",
args,
"run_phase_"+cfg.CB_CLUSTER_TAG,
broker = remoteIP,
exchange = "default",
routing_key = cfg.CB_CLUSTER_TAG+".run.phase")
# get taskID of phase running on remote broker
taskId = None
while taskId is None and retry > 0:
time.sleep(2)
taskId = rabbitHelper.getMsg(rcq)
taskIds.append((remoteIP, taskId))
retry = retry - 1
return taskIds
def import_template(args):
val = None
if args.type == "json":
json_val = {}
for kv in args.kvpairs:
pair = '{%s}' % kv
try:
pair = json.loads(pair)
json_val.update(pair)
except ValueError as ex:
print "ERROR: Unable to encode as valid json: %s " % kv
print "make sure strings surrounded by double quotes"
return
val = json_val
#TODO binary blobs
template = { "name" : args.name,
"ttl" : args.ttl,
"flags" : args.flags,
"cc_queues" : args.cc_queues,
"size" : args.size,
"kv" : val}
rabbitHelper = RabbitHelper(args.broker)
rabbitHelper.putMsg("workload_template", json.dumps(template))
def run_workload(args):
workload = {}
if args.name != None:
# TODO: read in workload params from saved store
# workload.update(cached_workload)
pass
if args.wait is not None:
args.wait = conv_to_secs(args.wait)
workload = { "bucket" : args.bucket,
"password" : args.password,
"ops_per_sec" : args.ops,
"create_perc" : args.create,
"update_perc" : args.update,
"get_perc" : args.get,
"del_perc" : args.delete,
"exp_perc" : args.expire,
"miss_perc" : args.miss,
"ttl" : args.ttl,
"cc_queues" : args.cc_queues,
"consume_queue" : args.consume_queue,
"postconditions" : args.postcondition,
"preconditions" : args.precondition,
"wait" : args.wait,
"template" : args.template}
cluster = args.cluster
rabbitHelper = RabbitHelper(args.broker)
workload['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg("workload_"+cluster, json.dumps(workload))
receiveResponse(rabbitHelper, workload['rcq'])
def getKeyMapFromRemoteQueue(self, requeue = True):
key_map = None
mq = RabbitHelper()
if mq.qsize(self.ccq) > 0:
try:
key_map = mq.getJsonMsg(self.ccq, requeue = requeue )
except Exception:
pass
return key_map
def requeueNonDeletedKeys(self):
rabbitHelper = RabbitHelper()
task_type = 'app.sdk_client_tasks.mdelete'
# requeue pending delete keys so that they may be deleted in another workload
while rabbitHelper.qsize(self.task_queue) > 0:
task_set = rabbitHelper.getJsonMsg(self.task_queue)
if len(task_set) > 0:
keys = [task['args'] for task in task_set \
if task['task'] == task_type]
if len(keys) > 0:
# put back on to consume_queue
msg = json.dumps(keys[0][0])
rabbitHelper.putMsg(self.consume_queue, msg)
try:
# delete task queue
rabbitHelper.delete(self.task_queue)
# delete consume queue if it was a miss_queue
if self.miss_queue is not None and self.consume_queue is not None:
rabbitHelper.delete(self.consume_queue)
except:
pass
def perform_query_tasks(args):
queryMsg = {'queries_per_sec' : args.queries_per_sec,
'ddoc' : args.ddoc,
'view' : args.view,
'bucket' : args.bucket,
'password' : args.password}
cluster = args.cluster
rabbitHelper = RabbitHelper(args.broker)
queryMsg['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg('query_'+cluster, json.dumps(queryMsg))
receiveResponse(rabbitHelper, queryMsg['rcq'])
def perform_admin_tasks(args):
actions = {'rebalance_in': args.rebalance_in,
'rebalance_out': args.rebalance_out,
'failover': args.failover,
'soft_restart': args.soft_restart,
'hard_restart': args.hard_restart,
'only_failover': args.only_failover
}
#TODO: Validate the user inputs, before passing to rabbit
print actions
rabbitHelper = RabbitHelper(args.broker)
rabbitHelper.putMsg("admin_tasks", json.dumps(actions))
def perform_admin_tasks(args):
actions = {
'rebalance_in': args.rebalance_in,
'rebalance_out': args.rebalance_out,
'failover': args.failover,
'soft_restart': args.soft_restart,
'hard_restart': args.hard_restart,
'only_failover': args.only_failover
}
#TODO: Validate the user inputs, before passing to rabbit
print actions
rabbitHelper = RabbitHelper(args.broker)
rabbitHelper.putMsg("admin_tasks", json.dumps(actions))
def perform_admin_tasks(args):
actions = {'rebalance_in': args.rebalance_in,
'rebalance_out': args.rebalance_out,
'failover': args.failover,
'soft_restart': args.soft_restart,
'hard_restart': args.hard_restart,
'only_failover': args.only_failover
}
cluster = args.cluster
#TODO: Validate the user inputs, before passing to rabbit
rabbitHelper = RabbitHelper(args.broker, cluster)
actions['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg("admin_"+cluster, json.dumps(actions))
receiveResponse(rabbitHelper, actions['rcq'])
def perform_xdcr_tasks(args):
xdcrMsg = {'dest_cluster_ip': args.dest_cluster_ip,
'dest_cluster_rest_username': args.dest_cluster_username,
'dest_cluster_rest_pwd': args.dest_cluster_pwd,
'dest_cluster_name': args.dest_cluster_name,
'replication_type': args.replication_type,
}
cluster = args.cluster
#TODO: Validate the user inputs, before passing to rabbit
print xdcrMsg
rabbitHelper = RabbitHelper(args.broker, cluster)
xdcrMsg['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg("xdcr_"+cluster, json.dumps(xdcrMsg))
receiveResponse(rabbitHelper, xdcrMsg['rcq'])
def perform_xdcr_tasks(args):
xdcrMsg = {'dest_cluster_ip': args.dest_cluster_ip,
'dest_cluster_rest_username': args.dest_cluster_username,
'dest_cluster_rest_pwd': args.dest_cluster_pwd,
'dest_cluster_name': args.dest_cluster_name,
'replication_type': args.replication_type,
}
cluster = args.cluster
#TODO: Validate the user inputs, before passing to rabbit
print xdcrMsg
rabbitHelper = RabbitHelper(args.broker)
xdcrMsg['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg("xdcr_"+cluster, json.dumps(xdcrMsg))
receiveResponse(rabbitHelper, xdcrMsg['rcq'])
def perform_admin_tasks(args):
actions = {'rebalance_in': args.rebalance_in,
'rebalance_out': args.rebalance_out,
'failover': args.failover,
'soft_restart': args.soft_restart,
'hard_restart': args.hard_restart,
'only_failover': args.only_failover
}
cluster = args.cluster
#TODO: Validate the user inputs, before passing to rabbit
rabbitHelper = RabbitHelper(args.broker)
actions['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg("admin_"+cluster, json.dumps(actions))
receiveResponse(rabbitHelper, actions['rcq'])
def flushq(self, flush_hotkeys = False):
mq = RabbitHelper()
if self.ccq is not None:
logging.info("[Thread %s] flushing %s items to %s" %
(self.name, self.memq.qsize(), self.ccq))
# declare queue
mq.declare(self.ccq)
# empty the in memory queue
while self.memq.empty() == False:
try:
msg = self.memq.get_nowait()
msg = json.dumps(msg)
mq.putMsg(self.ccq, msg)
except queue.Empty:
pass
# hot keys
if flush_hotkeys and (len(self.hotkey_batches) > 0):
# try to put onto remote queue
queue = self.consume_queue or self.ccq
if queue is not None:
key_map = {'start' : self.hotkey_batches[0][0],
'end' : self.hotkey_batches[-1][-1]}
msg = json.dumps(key_map)
mq.putMsg(queue, msg)
self.hotkey_batches = []
def getKeyMapFromRemoteQueue(self, requeue = True):
key_map = None
mq = RabbitHelper()
# try to fetch from consume queue and
# fall back to ccqueue
queue = self.consume_queue
if queue is None or mq.qsize(queue) == 0:
queue = self.ccq
if mq.qsize(queue) > 0:
try:
key_map = mq.getJsonMsg(queue, requeue = requeue )
except Exception:
pass
return key_map
def getKeyMapFromRemoteQueue(self, requeue=True):
key_map = None
mq = RabbitHelper()
# try to fetch from consume queue and
# fall back to ccqueue
queue = self.consume_queue
if queue is None or mq.qsize(queue) == 0:
queue = self.ccq
if mq.qsize(queue) > 0:
try:
key_map = mq.getJsonMsg(queue, requeue=requeue)
except Exception:
pass
return key_map
def run_systemtest(args):
cluster = args.cluster
rabbitHelper = RabbitHelper(args.broker, cluster)
test = {'suffix': args.filesuffix}
if args.fromfile is not None:
# load json config
json_data = open(args.fromfile)
msg = json.load(json_data)
elif args.name is not None:
msg = {"localtestname": args.name}
test.update(msg)
test['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg('systest_manager_' + cluster, json.dumps(test))
receiveResponse(rabbitHelper, test['rcq'])
def run_systemtest(args):
cluster = args.cluster
rabbitHelper = RabbitHelper(args.broker, cluster)
test = {'suffix' : args.filesuffix}
if args.fromfile is not None:
# load json config
json_data = open(args.fromfile)
msg = json.load(json_data)
elif args.name is not None:
msg = { "localtestname" : args.name }
test.update(msg)
test['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg('systest_manager_'+cluster, json.dumps(test))
receiveResponse(rabbitHelper, test['rcq'])
def perform_query_tasks(args):
queryMsg = {'queries_per_sec' : args.queries_per_sec,
'ddoc' : args.ddoc,
'view' : args.view,
'bucket' : args.bucket,
'password' : args.password,
'include_filters' : args.include_filters,
'exclude_filters' : args.exclude_filters,
'startkey' : args.startkey,
'endkey' : args.endkey,
'startkey_docid' : args.startkey_docid,
'endkey_docid' : args.endkey_docid,
'limit' : args.limit,
'indexed_key' : args.indexed_key}
cluster = args.cluster
rabbitHelper = RabbitHelper(args.broker, cluster)
queryMsg['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg('query_'+cluster, json.dumps(queryMsg))
receiveResponse(rabbitHelper, queryMsg['rcq'])
def main():
args = parser.parse_args()
CB_CLUSTER_TAG = args.cluster
exchange = CB_CLUSTER_TAG + "consumers"
# setup to consume messages from worker
mq = RabbitHelper()
mq.exchange_declare(exchange, "fanout")
queue = mq.declare()
queue_name = queue[0]
# bind to exchange
mq.bind(exchange, queue_name)
mq.putMsg('', 'init', exchange)
# consume messages
channel, conn = mq.channel()
channel.basic_consume(callback=init, queue=queue_name, no_ack=True)
while True:
conn.drain_events()
def worker_init():
# cleanup queues
rabbitHelper = RabbitHelper()
cached_queues = WorkloadCacher().queues + TemplateCacher().cc_queues
test_queues = [
"workload", "workload_template", "admin_tasks", "xdcr_tasks"
] + cached_queues
for queue in test_queues:
try:
if rabbitHelper.qsize(queue) > 0:
print "Purge Queue: " + queue + " " + str(
rabbitHelper.qsize(queue))
rabbitHelper.purge(queue)
except Exception as ex:
print ex
cacheClean()
# kill old background processes
kill_procs = ["sdkserver"]
for proc in kill_procs:
os.system("ps aux | grep %s | awk '{print $2}' | xargs kill" % proc)
# start sdk servers
os.system("ruby sdkserver.rb &")
os.system("python sdkserver.py &")
# make sure logdir exists
os.system("mkdir -p " + cfg.LOGDIR)
def __setattr__(self, name, value):
super(Workload, self).__setattr__(name, value)
# auto cache workload when certain attributes change
# if object has been fully setup
if name in Workload.AUTOCACHEKEYS and self.initialized:
ObjCacher().store(CacheHelper.WORKLOADCACHEKEY, self)
# check if workload is being deactivated
if name == "active" and self.active == False and self.initialized:
msg = {'active': False, 'id': self.id}
RabbitHelper().putMsg('', json.dumps(msg), EXCHANGE)
logger.error("kill task %s" % self.id)
def perform_query_tasks(args):
queryMsg = {
'queries_per_sec': args.queries_per_sec,
'ddoc': args.ddoc,
'view': args.view,
'bucket': args.bucket,
'password': args.password,
'include_filters': args.include_filters,
'exclude_filters': args.exclude_filters,
'startkey': args.startkey,
'endkey': args.endkey,
'startkey_docid': args.startkey_docid,
'endkey_docid': args.endkey_docid,
'limit': args.limit,
'indexed_key': args.indexed_key
}
cluster = args.cluster
rabbitHelper = RabbitHelper(args.broker, cluster)
queryMsg['rcq'] = getResponseQueue(rabbitHelper)
rabbitHelper.putMsg('query_' + cluster, json.dumps(queryMsg))
receiveResponse(rabbitHelper, queryMsg['rcq'])
class Consumer(Greenlet):
def __init__(self, queue):
Greenlet.__init__(self)
self.queue = queue
self.cb = GConnection(bucket='default')
self.conn = RabbitHelper()
def _run(self):
while True:
data = self.conn.getMsg("gvset")
if data:
do(self.cb, data)
def main():
args = parser.parse_args()
CB_CLUSTER_TAG = args.cluster
exchange = CB_CLUSTER_TAG+"consumers"
# setup to consume messages from worker
mq = RabbitHelper()
mq.exchange_declare(exchange, "fanout")
queue = mq.declare()
queue_name = queue[0]
# bind to exchange
mq.bind(exchange, queue_name)
mq.putMsg('', 'init', exchange)
# consume messages
channel, conn = mq.channel()
channel.basic_consume(callback = init, queue = queue_name, no_ack = True)
while True:
conn.drain_events()
def flushq(self, flush_hotkeys=False):
if self.standalone:
return
mq = RabbitHelper()
if self.ccq is not None:
logging.info("[Thread %s] flushing %s items to %s" %
(self.name, self.memq.qsize(), self.ccq))
# declare queue
mq.declare(self.ccq)
# empty the in memory queue
while self.memq.empty() == False:
try:
msg = self.memq.get_nowait()
msg = json.dumps(msg)
mq.putMsg(self.ccq, msg)
except queue.Empty:
pass
# hot keys
if flush_hotkeys and (len(self.hotkey_batches) > 0):
# try to put onto remote queue
queue = self.consume_queue or self.ccq
if queue is not None:
key_map = {
'start': self.hotkey_batches[0][0],
'end': self.hotkey_batches[-1][-1]
}
msg = json.dumps(key_map)
mq.putMsg(queue, msg)
self.hotkey_batches = []
def flushq(self):
if self.ccq is not None:
# declare queue
mq = RabbitHelper()
mq.declare(self.ccq)
while self.memq.empty() == False:
try:
msg = self.memq.get_nowait()
msg = json.dumps(msg)
mq.putMsg(self.ccq, msg)
except queue.Empty:
pass
# hot keys
if len(self.hotkeys) > 0:
key_map = {'start' : self.hotkeys[0],
'end' : self.hotkeys[-1]}
msg = json.dumps(key_map)
mq.putMsg(self.ccq, msg)
def __init__(self, queue):
Greenlet.__init__(self)
self.queue = queue
self.cb = GConnection(bucket='default')
self.conn = RabbitHelper()
def run(workload):
workload.active = True
rabbitHelper = RabbitHelper()
sdk_queue_key = "sdk_consumer.*"
# read doc template
template = Template.from_cache(str(workload.template))
if template is None:
logger.error("no doc template imported")
return
consumer_template = copy.deepcopy(template)
bucket = str(workload.bucket)
password = str(workload.password)
active_hosts = None
clusterStatus = CacheHelper.clusterstatus(cfg.CB_CLUSTER_TAG + "_status")
if clusterStatus is not None:
active_hosts = clusterStatus.get_all_hosts()
if workload.cc_queues is not None:
# override template attribute with workload
consumer_template.cc_queues = workload.cc_queues
if len(workload.indexed_keys) > 0:
template.indexed_keys = workload.indexed_keys
ops_sec = workload.ops_per_sec
# modify ops by number of consumers
num_consumers = rabbitHelper.numExchangeQueues(cfg.CB_CLUSTER_TAG,
EXCHANGE)
if num_consumers == 0:
logger.error("No sdkclients running")
return
ops_sec = int(ops_sec) / num_consumers
create_count = int(ops_sec * workload.create_perc / 100)
update_count = int(ops_sec * workload.update_perc / 100)
get_count = int(ops_sec * workload.get_perc / 100)
del_count = int(ops_sec * workload.del_perc / 100)
exp_count = int(ops_sec * workload.exp_perc / 100)
consume_queue = workload.consume_queue
ttl = workload.ttl
miss_queue = workload.miss_queue
miss_perc = workload.miss_perc
# broadcast to sdk_consumers
msg = {
'bucket': bucket,
'id': workload.id,
'password': password,
'template': consumer_template.__dict__,
'ops_sec': ops_sec,
'create_count': create_count,
'update_count': update_count,
'get_count': get_count,
'del_count': del_count,
'exp_count': exp_count,
'consume_queue': consume_queue,
'ttl': ttl,
'miss_perc': miss_perc,
'active': True,
'active_hosts': active_hosts
}
rabbitHelper.putMsg('', json.dumps(msg), EXCHANGE)
logger.error("start task sent to %s consumers" % num_consumers)
import os
import time
from rabbit_helper import RabbitHelper
from cache import WorkloadCacher, TemplateCacher, BucketStatusCacher, cacheClean
# cleanup queues
rabbitHelper = RabbitHelper()
cached_queues = WorkloadCacher().queues + TemplateCacher().cc_queues
test_queues = ["workload", "workload_template"] + cached_queues
for queue in test_queues:
try:
if rabbitHelper.qsize(queue) > 0:
print "Purge Queue: " + queue + " " + str(rabbitHelper.qsize(queue))
rabbitHelper.purge(queue)
except Exception as ex:
pass
cacheClean()
# kill+start sdk's
os.system("ps aux | grep sdkserver | awk '{print $2'} | xargs kill")
os.system("ruby sdkserver.rb &")
os.system("python sdkserver.py &")
queues = set(CacheHelper.queues())
# cleaning up seriesly database (fast and slow created by cbtop)
if cfg.SERIESLY_IP != '':
from seriesly import Seriesly
os.system("curl -X DELETE http://{0}:3133/fast".format(
cfg.SERIESLY_IP))
os.system("curl -X DELETE http://{0}:3133/slow".format(
cfg.SERIESLY_IP))
os.system("curl -X DELETE http://{0}:3133/event".format(
cfg.SERIESLY_IP))
os.system("curl -X DELETE http://{0}:3133/atop".format(
cfg.SERIESLY_IP))
seriesly = Seriesly(cfg.SERIESLY_IP, 3133)
seriesly.create_db('event')
for q_ in queues:
try:
RabbitHelper().delete(q_)
print "Cleanup Queue: %s" % q_
except Exception as ex:
pass
# clean up cache
CacheHelper.cacheClean()
# start sdk server
os.system("python sdkserver.py &")
import os
import time
from rabbit_helper import RabbitHelper
from cache import WorkloadCacher, TemplateCacher, BucketStatusCacher, cacheClean
# cleanup queues
rabbitHelper = RabbitHelper()
cached_queues = WorkloadCacher().queues + TemplateCacher().cc_queues
test_queues = ["workload","workload_template"] + cached_queues
for queue in test_queues:
try:
if rabbitHelper.qsize(queue) > 0:
print "Purge Queue: "+queue +" "+ str(rabbitHelper.qsize(queue))
rabbitHelper.purge(queue)
except Exception as ex:
pass
cacheClean()
# kill+start sdk's
os.system("ps aux | grep sdkserver | awk '{print $2'} | xargs kill")
os.system("ruby sdkserver.rb &")
os.system("python sdkserver.py &")
import argparse
import json
from rabbit_helper import RabbitHelper
parser = argparse.ArgumentParser(description='CB System Test Tool')
subparser = parser.add_subparsers(dest="subparsers")
rabbitHelper = RabbitHelper()
def add_modifier_args(parser):
parser.add_argument("--cc_queues", nargs='+', help="queues to copy created keys into")
parser.add_argument("--consume_queue",help="queue with keys to get/update/delete")
parser.add_argument("--precondition", help="required stat or cluster state required before running workload")
parser.add_argument("--postcondition",help="required stat or cluster state required to complete workload")
parser.add_argument("--wait", nargs=3, help="time to wait before starting workload: <hour> <min> <sec>", metavar = ('HOUR','MIN','SEC'), type=int)
parser.add_argument("--expires",nargs=3, help="time to wait before terminating workload: <hour> <min> <sec>", metavar = ('HOUR','MIN','SEC'), type=int)
def add_template_parser(parent):
parser = parent.add_parser("template")
parser.add_argument("--name", help="template name", required = True)
parser.add_argument("--ttl", default=0, help="document expires time")
parser.add_argument("--flags", default=0, help="document create flags")
parser.add_argument("--cc_queues",nargs='+', help="queues to copy created keys into")
parser.add_argument("--kvpairs", nargs='+', help="list of kv items i.e=> state:ca,age:28,company:cb")
parser.add_argument("--type", help="json/non-json default is json", default="json")
parser.add_argument("--size", help="size of documents. padding is used if necessary")
#TODO parser.add_argument("--blobs", nargs='+', help="data strings for non-json docs")
parser.set_defaults(handler=import_template)
# as seriesly db
if "--purge" in sys.argv:
queues = set(CacheHelper.queues())
# cleaning up seriesly database (fast and slow created by cbtop)
if cfg.SERIESLY_IP != '':
from seriesly import Seriesly
seriesly = Seriesly(cfg.SERIESLY_IP, 3133)
dbs = seriesly.list_dbs()
for db in dbs:
seriesly.drop_db(db)
seriesly.create_db('event')
for q_ in queues:
try:
RabbitHelper().delete(q_)
print("Cleanup Queue: %s" % q_)
except Exception as ex:
pass
# clean up cache
CacheHelper.cacheClean()
# start local consumer
exchange = cfg.CB_CLUSTER_TAG + "consumers"
RabbitHelper().exchange_declare(exchange, "fanout")
os.system("python consumer.py &")
os.system("ulimit -n 10240")
def __init__(self, name, task, e):
threading.Thread.__init__(self)
self.name = name
self.i = 0
self.op_factor = CLIENTSPERPROCESS * PROCSPERTASK
self.ops_sec = task['ops_sec']
self.bucket = task['bucket']
self.password = task['password']
self.template = task['template']
self.default_tsizes = [128, 256]
self.create_count = task['create_count'] / self.op_factor
self.update_count = task['update_count'] / self.op_factor
self.get_count = task['get_count'] / self.op_factor
self.del_count = task['del_count'] / self.op_factor
self.exp_count = task['exp_count'] / self.op_factor
self.ttl = task['ttl']
self.miss_perc = task['miss_perc']
self.active_hosts = task['active_hosts']
self.batch_size = 5000
self.memq = queue.Queue()
self.consume_queue = task['consume_queue']
self.standalone = task['standalone']
self.ccq = None
self.hotkey_batches = []
if self.consume_queue is not None:
RabbitHelper().declare(self.consume_queue)
if task['template']['cc_queues']:
self.ccq = str(
task['template']['cc_queues'][0]) #only supporting 1 now
RabbitHelper().declare(self.ccq)
if self.batch_size > self.create_count:
self.batch_size = self.create_count
self.active_hosts = task['active_hosts']
if not self.active_hosts:
self.active_hosts = [cfg.COUCHBASE_IP]
addr = task['active_hosts'][random.randint(0,
len(self.active_hosts) -
1)].split(':')
host = addr[0]
port = 8091
if len(addr) > 1:
port = addr[1]
self.e = e
self.cb = None
self.isterminal = False
self.done = False
try:
endpoint = "%s:%s/%s" % (host, port, self.bucket)
self.cb = Bucket(endpoint, password=self.password)
except Exception as ex:
logging.error("[Thread %s] cannot reach %s" %
(self.name, endpoint))
logging.error(ex)
self.isterminal = True
logging.info("[Thread %s] started for workload: %s" %
(self.name, task['id']))
def run(workload):
workload.active = True
rabbitHelper = RabbitHelper()
sdk_queue_key = "sdk_consumer.*"
# read doc template
template = Template.from_cache(str(workload.template))
if template is None:
logger.error("no doc template imported")
return
consumer_template = copy.deepcopy(template)
bucket = str(workload.bucket)
password = str(workload.password)
active_hosts = None
clusterStatus = CacheHelper.clusterstatus(cfg.CB_CLUSTER_TAG+"_status")
if clusterStatus is not None:
active_hosts = clusterStatus.get_all_hosts()
if workload.cc_queues is not None:
# override template attribute with workload
consumer_template.cc_queues = workload.cc_queues
if len(workload.indexed_keys) > 0:
template.indexed_keys = workload.indexed_keys
ops_sec = workload.ops_per_sec
# modify ops by number of consumers
num_consumers = rabbitHelper.numExchangeQueues(cfg.CB_CLUSTER_TAG, EXCHANGE)
if num_consumers == 0:
logger.error("No sdkclients running")
return
ops_sec = int(ops_sec)/num_consumers
create_count = int(ops_sec * workload.create_perc/100)
update_count = int(ops_sec * workload.update_perc/100)
get_count = int(ops_sec * workload.get_perc/100)
del_count = int(ops_sec * workload.del_perc/100)
exp_count = int(ops_sec * workload.exp_perc/100)
consume_queue = workload.consume_queue
ttl = workload.ttl
miss_queue = workload.miss_queue
miss_perc = workload.miss_perc
# broadcast to sdk_consumers
msg = {'bucket' : bucket,
'id' : workload.id,
'password' : password,
'template' : consumer_template.__dict__,
'ops_sec' : ops_sec,
'create_count' : create_count,
'update_count' : update_count,
'get_count' : get_count,
'del_count' : del_count,
'exp_count' : exp_count,
'consume_queue' : consume_queue,
'ttl' : ttl,
'miss_perc' : miss_perc,
'active' : True,
'active_hosts' : active_hosts}
rabbitHelper.putMsg('', json.dumps(msg), EXCHANGE)
logger.error("start task sent to %s consumers" % num_consumers)
