def addTask(self, bucket, taskid, workload):
newPair = (taskid, workload)
if bucket not in self.history:
self.history[bucket] = {"tasks": [newPair]}
else:
self.history[bucket]["tasks"].append(newPair)
ObjCacher().store(CacheHelper.BUCKETSTATUSCACHEKEY, self)
def setupCacheMissQueues(workload):
""" assuming misses will come from keys in
consume_queue or cc_queue.
so make location where keys were going
to be read the miss queue and set
consume queue to location where new keys are
being generated.
Only required that at least a cc_queue
with miss items is provided"""
# make another cc_queue to put only hot items
new_cc_queue = None
if workload.cc_queues is None:
new_cc_queue = workload.id + "__hot__"
workload.cc_queues = [new_cc_queue]
# delete new cc_queue if it exists
try:
rabbitHelper.delete(new_cc_queue)
except:
pass # queue already deleted
else:
new_cc_queue = workload.cc_queues[0]
# move old consume queue to miss queue
if workload.consume_queue is not None:
workload.miss_queue = workload.consume_queue
# make new cc_queue the consume queue
workload.consume_queue = new_cc_queue
# save changes
ObjCacher().store(CacheHelper.WORKLOADCACHEKEY, workload)
def __setattr__(self, name, value):
super(ActiveTaskChecker, self).__setattr__(name, value)
# auto cache object when certain keys change
if self.initialized and name in ("initialized", "task_started",
"empty_stat_count"):
ObjCacher().store(CacheHelper.ACTIVETASKCACHEKEY, self)
def __init__(self, params):
self.name = params["name"]
self.id = self.name
self.ttl = params["ttl"]
self.flags = params["flags"]
self.cc_queues = params["cc_queues"]
self.kv = params["kv"]
self.size = params["size"]
# cache
ObjCacher().store(CacheHelper.TEMPLATECACHEKEY, self)
def __setattr__(self, name, value):
super(Workload, self).__setattr__(name, value)
# auto cache workload when certain attributes change
# if object has been fully setup
if name in Workload.AUTOCACHEKEYS and self.initialized:
ObjCacher().store(CacheHelper.WORKLOADCACHEKEY, self)
# check if workload is being deactivated
if name == "active" and self.active == False and self.initialized:
self.requeueNonDeletedKeys()
def __setattr__(self, name, value):
super(Workload, self).__setattr__(name, value)
# auto cache workload when certain attributes change
# if object has been fully setup
if name in Workload.AUTOCACHEKEYS and self.initialized:
ObjCacher().store(CacheHelper.WORKLOADCACHEKEY, self)
# check if workload is being deactivated
if name == "active" and self.active == False and self.initialized:
msg = {'active': False, 'id': self.id}
RabbitHelper().putMsg('', json.dumps(msg), EXCHANGE)
logger.error("kill task %s" % self.id)
def update(self, recent_key, recent_id):
if self.startkey is None or recent_key < self.startkey:
self.startkey = recent_key
self.startkey_docid = recent_id
if self.endkey is None or recent_key > self.endkey:
self.endkey = recent_key
self.endkey_docid = recent_id
self.recentkey = recent_key
self.recentkey_docid = recent_id
ObjCacher().store(CacheHelper.QBUILDCACHEKEY, self)
def __init__(self, params):
logger.error(params)
self.name = params["name"]
self.id = self.name
self.ttl = params["ttl"]
self.flags = params["flags"]
self.cc_queues = params["cc_queues"]
self.kv = params["kv"]
self.size = params.get("size") or ['128']
self.size = map(int, self.size)
self.indexed_keys = []
# cache
ObjCacher().store(CacheHelper.TEMPLATECACHEKEY, self)
def update_node_stats(node_stats, sample):
for key in sample.keys():
if key != 'ip':
if key not in node_stats.samples:
node_stats.samples[key] = []
try:
val = float(re.sub(r'[^\d.]+', '', sample[key]))
node_stats.samples[key].append(val)
ObjCacher().store(CacheHelper.NODESTATSCACHEKEY, node_stats)
except Exception as ex:
logger.error("Error saving data for (%s,%s): %s" %
(key, sample[key], ex))
def updateIndexKeys(self, key):
template = Template.from_cache(str(self.template))
# update workload with information about which keys being index
if key is not None:
# when indexed key does not exist in kv pair do not update
if key in template.kv:
# do not update if we are already traking index key
if key not in self.indexed_keys:
# update and cache workload object
self.indexed_keys.append(key)
ObjCacher().store(CacheHelper.WORKLOADCACHEKEY, self)
else:
logger.error(
"key: '%s' does not exist in kvpair. Smart querying disabled"
% key)
def updateClusterStatus(ignore_result=True):
done = False
clusterStatus = CacheHelper.clusterstatus(cfg.CB_CLUSTER_TAG+"_status") or\
ClusterStatus()
# check cluster nodes
cached_nodes = clusterStatus.nodes
new_cached_nodes = []
for node in cached_nodes:
# get an active node
if clusterStatus.http_ping_node(node) is not None:
# get remaining nodes
active_nodes = clusterStatus.get_cluster_nodes(node)
# populate cache with healthy nodes
for active_node in active_nodes:
if active_node.status == 'healthy':
new_cached_nodes.append(active_node)
break
if len(new_cached_nodes) > 0:
# check for update
new_node_list = ["%s:%s" % (n.ip, n.port) for n in new_cached_nodes]
if len(new_node_list) != len(cached_nodes) or\
len(set(clusterStatus.get_all_hosts()).intersection(new_node_list)) !=\
len(cached_nodes):
clusterStatus.nodes = new_cached_nodes
clusterStatus.update_orchestrator()
else:
clusterStatus.orchestrator = None
ObjCacher().delete(CacheHelper.CLUSTERSTATUSKEY, clusterStatus)
def __setattr__(self, name, value):
super(QueryWorkload, self).__setattr__(name, value)
# cache when active key mutated
if name == 'active':
ObjCacher().store(CacheHelper.QUERYCACHEKEY, self)
def __setattr__(self, name, value):
# auto cache changes made to this object
super(ClusterStatus, self).__setattr__(name, value)
if self.initialized:
ObjCacher().store(CacheHelper.CLUSTERSTATUSKEY, self)
def __setattr__(self, name, value):
super(NodeStats, self).__setattr__(name, value)
ObjCacher().store(CacheHelper.NODESTATSCACHEKEY, self)
def from_cache(id_):
return ObjCacher().instance(CacheHelper.BUCKETSTATUSCACHEKEY, id_)
def __setattr__(self, name, value):
super(BucketStatus, self).__setattr__(name, value)
ObjCacher().store(CacheHelper.BUCKETSTATUSCACHEKEY, self)
def from_cache(id_):
return ObjCacher().instance(CacheHelper.NODESTATSCACHEKEY, id_)
def from_cache(id_):
return ObjCacher().instance(CacheHelper.TEMPLATECACHEKEY, id_)
def __setattr__(self, name, value):
super(Workload, self).__setattr__(name, value)
# cache when active key mutated
if name == 'active' or name == 'postconditions':
ObjCacher().store(CacheHelper.WORKLOADCACHEKEY, self)
import copy
import os
sys.path = ["../"] + sys.path
import unittest
import logger
from membase.api.rest_client import RestConnection, Bucket, RestHelper
from couchbase.cluster import Cluster
from TestInput import TestInputSingleton
from membase.helper.bucket_helper import BucketOperationHelper
from membase.helper.cluster_helper import ClusterOperationHelper
from cache import ObjCacher, CacheHelper
import testcfg as cfg
from app.workload_manager import ClusterStatus
ObjCacher().clear(CacheHelper.CLUSTERSTATUSKEY)
class initialize(unittest.TestCase):
def setUp(self):
self._log = logger.Logger.get_logger()
self._input = TestInputSingleton.input
self._clusters_dic = self._input.clusters
self._clusters_keys_olst = range(len(self._clusters_dic))
try:
self._num_initial_nodes = self._input.param("initial_nodes",
'1').split(',')
except:
self._num_initial_nodes = [self._input.param("initial_nodes", '1')]
self._buckets = []
self._default_bucket = self._input.param("default_bucket", False)
def from_cache():
id_ = cfg.CB_CLUSTER_TAG + "active_task_status"
return ObjCacher().instance(CacheHelper.ACTIVETASKCACHEKEY, id_)
def from_cache(id_):
return ObjCacher().instance(CacheHelper.QBUILDCACHEKEY, id_)
def from_cache(id_):
return ObjCacher().instance(CacheHelper.WORKLOADCACHEKEY, id_)
