def direct_mc_bin_client(self, server, bucket, timeout=30):
# USE MC BIN CLIENT WHEN NOT USING SDK CLIENT
rest = RestConnection(server)
node = None
try:
node = rest.get_nodes_self()
except ValueError as e:
self.log.info("could not connect to server {0}, will try scanning all nodes".format(server))
if not node:
nodes = rest.get_nodes()
for n in nodes:
if n.ip == server.ip and n.port == server.port:
node = n
if isinstance(server, dict):
self.log.info("dict:{0}".format(server))
self.log.info("creating direct client {0}:{1} {2}".format(server["ip"], node.memcached, bucket))
else:
self.log.info("creating direct client {0}:{1} {2}".format(server.ip, node.memcached, bucket))
RestHelper(rest).vbucket_map_ready(bucket, 60)
vBuckets = RestConnection(server).get_vbuckets(bucket)
if isinstance(server, dict):
client = MemcachedClient(server["ip"], node.memcached, timeout=timeout)
else:
client = MemcachedClient(server.ip, node.memcached, timeout=timeout)
if vBuckets != None:
client.vbucket_count = len(vBuckets)
else:
client.vbucket_count = 0
bucket_info = rest.get_bucket(bucket)
return client
class SubdocSinglePathTests(SubdocBaseTest):
def setUp(self):
super(SubdocSinglePathTests, self).setUp()
self.client = MemcachedClient(host=self.server.ip)
self.jsonSchema = {
"id": "0",
"number": 0,
"array": [],
"child": {},
"isDict": True,
"padding": None
}
def tearDown(self):
super(SubdocSinglePathTests, self).tearDown()
def insertBinaryDocument(self, key):
pass
def _createNestedJson(self, key, dict):
if dict['levels'] == 0:
return
if dict['doc'] == {}:
dict['doc'] = copy.copy(self.jsonSchema)
else:
dict['doc']['child'] = copy.copy(self.jsonSchema)
dict['doc']['child']['array'] = []
dict['doc'] = dict['doc']['child']
dict['doc']['id'] = key
dict['doc']['number'] = dict['levels']
for level in range(0, dict['levels']):
dict['doc']['array'].append(level)
return self._createNestedJson(key, {
'doc': dict['doc'],
'levels': dict['levels'] - 1
})
def insertJsonDocument(self, key, levels, expiry, size=512):
dict = {'doc': {}, 'levels': levels}
self._createNestedJson(key, dict)
jsonDump = json.dumps(dict['doc'])
self.client.set(key, expiry, 0, jsonDump)
def deleteDoc(self, key):
self.client.delete(key)
def _load_all_docs(self, key=None, levels=0, num_items=0):
dict = {'doc': {}, 'levels': levels}
self._createNestedJson(key, dict)
template = dict['doc']
gen_load = SubdocDocumentGenerator(key,
template,
start=0,
end=num_items)
print("Inserting json data into bucket")
self._load_all_buckets(self.server, gen_load, "create", 0)
self._wait_for_stats_all_buckets([self.server])
def collect_vbucket_num_stats(self,servers, buckets):
"""
Method to extract the failovers stats given by cbstats tool
Paramters:
buckets: bucket informaiton
servers: server information
Returns:
Failover stats as follows:
if not collecting per node :: {bucket : [{key:value}]}
if collecting per node :: {bucket : {node:[{key:value}]}}
"""
active_bucketMap = {}
replica_bucketMap = {}
for bucket in buckets:
active_map_data = {}
replica_map_data = {}
for server in servers:
rest = RestConnection(server)
port = rest.get_memcached_port()
client = MemcachedClient(host=server.ip, port=port)
stats = client.stats('')
for key in stats.keys():
if key == 'vb_active_num':
active_map_data[server.ip] = int(stats[key])
if key == 'vb_replica_num':
replica_map_data[server.ip] = int(stats[key])
active_bucketMap[bucket.name] = active_map_data
replica_bucketMap[bucket.name] = replica_map_data
return active_bucketMap,replica_bucketMap
def __init__(self,
serverip="localhost",
port=11211,
bucket="default",
password=""):
self.client = MemcachedClient(serverip, port)
self.client.sasl_auth_plain(bucket, password)
def wait_for_warmup(host, port):
while True:
client = McdClient(host, port)
try:
response = client.stats()
if response['ep_degraded_mode'] == '0':
break
except:
pass
time.sleep(1)
class SubdocSinglePathTests(SubdocBaseTest):
def setUp(self):
super(SubdocSinglePathTests, self).setUp()
self.client = MemcachedClient(host=self.server.ip)
self.jsonSchema = {
"id" : "0",
"number" : 0,
"array" : [],
"child" : {},
"isDict" : True,
"padding": None
}
def tearDown(self):
super(SubdocSinglePathTests, self).tearDown()
def insertBinaryDocument(self, key):
pass
def _createNestedJson(self, key, dict):
if dict['levels'] == 0:
return
if dict['doc'] == {}:
dict['doc'] = copy.copy(self.jsonSchema)
else:
dict['doc']['child'] = copy.copy(self.jsonSchema)
dict['doc']['child']['array'] = []
dict['doc'] = dict['doc']['child']
dict['doc']['id'] = key
dict['doc']['number'] = dict['levels']
for level in xrange(0, dict['levels']):
dict['doc']['array'].append(level)
return self._createNestedJson(key, {'doc': dict['doc'], 'levels': dict['levels']-1})
def insertJsonDocument(self, key, levels, expiry, size=512):
dict = {'doc' : {}, 'levels' : levels }
self._createNestedJson(key, dict)
jsonDump = json.dumps(dict['doc'])
self.client.set(key, expiry, 0, jsonDump)
def deleteDoc(self, key):
self.client.delete(key)
def _load_all_docs(self, key=None, levels=0, num_items=0):
dict = {'doc' : {}, 'levels' : levels }
self._createNestedJson(key, dict)
template = dict['doc']
gen_load = SubdocDocumentGenerator(key, template,start=0, end=num_items)
print "Inserting json data into bucket"
self._load_all_buckets(self.server, gen_load, "create", 0)
self._wait_for_stats_all_buckets([self.server])
def collect_failovers_stats(self,buckets,servers,perNode = True):
"""
Method to extract the failovers stats given by cbstats tool
Paramters:
buckets: bucket informaiton
servers: server information
perNode: if set collect per node information else all
Returns:
Failover stats as follows:
if not collecting per node :: {bucket : [{key:value}]}
if collecting per node :: {bucket : {node:[{key:value}]}}
"""
bucketMap = {}
for bucket in buckets:
bucketMap[bucket.name] = {}
for bucket in buckets:
dataMap = {}
for server in servers:
rest = RestConnection(server)
port = rest.get_memcached_port()
client = MemcachedClient(host=server.ip, port=port)
stats = client.stats('failovers')
map_data = {}
num_map ={}
for o in stats.keys():
tokens = o.split(":")
vb = tokens[0]
key = tokens[1]
value = stats[o].split()
num = -1
if len(tokens) == 3:
vb = tokens[0]
num = int(tokens[1])
key = tokens[2]
if vb in map_data.keys() and (num == num_map[vb] or num > num_map[vb]):
map_data[vb][key] = value[0]
num_map[vb] = num
elif vb in map_data.keys() and key == "num_entries":
map_data[vb][key] = value[0]
elif vb not in map_data.keys():
m = {}
m[key] = value[0]
map_data[vb] = m
num_map[vb] = num
if perNode:
dataMap[server.ip] = map_data
else:
dataMap.update(map_data)
bucketMap[bucket.name] = dataMap
return bucketMap
def setUp(self):
super(SubdocSinglePathTests, self).setUp()
self.client = MemcachedClient(host=self.server.ip)
self.jsonSchema = {
"id" : "0",
"number" : 0,
"array" : [],
"child" : {},
"isDict" : True,
"padding": None
}
def collect_vbucket_stats(self,buckets,servers,collect_vbucket = True,collect_vbucket_seqno = True,collect_vbucket_details = True,perNode = True):
"""
Method to extract the vbuckets stats given by cbstats tool
Paramters:
buckets: bucket information
servers: server information
collect_vbucket: take vbucket type stats
collect_vbucket_seqno: take vbucket-seqno type stats
collect_vbucket_details: take vbucket-details type stats
perNode: if True collects data per node else takes a union across nodes
Returns:
The output can be in two formats
if we are doing per node data collection
Vbucket Information :: {bucket { node : [vbucket_seqno {key:value} U vbucket_details {key:value} U vbucket {key:value}]}}
if we are not doing per node data collection
Vbucket Information :: {bucket : [vbucket_seqno {key:value} U vbucket_details {key:value} U vbucket {key:value}]}
"""
bucketMap = {}
vbucket = []
vbucket_seqno = []
vbucket_details = []
for bucket in buckets:
bucketMap[bucket.name] = {}
for bucket in buckets:
dataMap = {}
for server in servers:
map_data = {}
rest = RestConnection(server)
port = rest.get_memcached_port()
client = MemcachedClient(host=server.ip, port=port)
if collect_vbucket:
vbucket=client.stats('vbucket')
self.createMapVbucket(vbucket,map_data)
if collect_vbucket_seqno:
vbucket_seqno=client.stats('vbucket-seqno')
self.createMapVbucket(vbucket_seqno,map_data)
if collect_vbucket_details:
vbucket_details=client.stats('vbucket-details')
self.createMapVbucket(vbucket_details,map_data)
if perNode:
dataMap[server.ip] = map_data
else:
dataMap.update(map_data)
bucketMap[bucket.name] = dataMap
return bucketMap
def wait_for_warmup(self, host, port):
if self.bucket_type == 'ephemeral': return
while True:
client = McdClient(host, port)
try:
client.bucket_select("default")
response = client.stats()
# check the old style or new style (as of 4.5) results
mode = response.get('ep_degraded_mode')
if mode is not None:
if mode == '0' or mode == 'false':
break
except Exception as ex:
pass
time.sleep(1)
def wait_for_warmup(self, host, port):
if self.bucket_type == 'ephemeral': return
while True:
client = McdClient(host, port)
try:
client.bucket_select("default")
response = client.stats()
# check the old style or new style (as of 4.5) results
mode = response.get('ep_degraded_mode')
if mode is not None:
if mode == '0' or mode == 'false':
break
except Exception as ex:
pass
time.sleep(1)
def __init__(self, testcase, bucket):
self.testcase = testcase
self.bucket = bucket
self.input = TestInputSingleton.input
self.servers = self.input.servers
self.master = self.servers[0]
self.rest = RestConnection(self.master)
self.log = logger.Logger.get_logger()
self.client = MemcachedClient(host=self.master.ip)
self.jsonSchema = {
"id" : "0",
"number" : 0,
"array" : [],
"child" : {},
"isDict" : True,
"padding": None
}
self.jsonSchema_longPath = {
"id" : "0",
"number" : 0,
"array12345678901234567890123456789" : [],
"child12345678901234567890123456789" : {},
"isDict" : True,
"padding": None
}
class MemcachedHelper(object):
def __init__(self,
serverip="localhost",
port=11211,
bucket="default",
password=""):
self.client = MemcachedClient(serverip, port)
self.client.sasl_auth_plain(bucket, password)
def write_one_json(self, key, doc):
count = 0
loaded = False
while count < 60 and not loaded:
try:
self.client.set(key, 0, 0, json.dumps(doc))
loaded = True
except MemcachedError as error:
if error.status == 134:
print "Memcached TMP_OOM, Retrying in 5 seconds..."
count += 1
time.sleep(5)
elif error.status == 7:
print "Not my vbucket error. If on MAC, please specify vbuckets as 64."
print "If rebalance is in progress. Please wait for it to finish.\n"
break
else:
print error
break
def read_one_json(self, key):
doc = ""
try:
_, _, doc = self.client.get(key)
except MemcachedError as error:
print error
return doc
def write_batch(self, batch_id, data):
print "Batch Insert not implemented for Memcached"
def collect_compare_dcp_stats(self,buckets,servers,perNode = True, stat_name = 'unacked_bytes', compare_value = 0, flow_control_buffer_size = 20971520, filter_list = []):
"""
Method to extract the failovers stats given by cbstats tool
Paramters:
buckets: bucket informaiton
servers: server information
stat_name: stat we are searching to compare
compare_value: the comparison value to be satisfied
Returns:
map of bucket informing if stat matching was satisfied/not satisfied
example:: unacked_bytes in dcp
"""
bucketMap = {}
for bucket in buckets:
bucketMap[bucket.name] = True
for bucket in buckets:
dataMap = {}
for server in servers:
rest = RestConnection(server)
port = rest.get_memcached_port()
client = MemcachedClient(host=server.ip, port=port)
stats = client.stats('dcp')
map_data = {}
for key in stats.keys():
filter = False
if stat_name in key:
for filter_key in filter_list:
if filter_key in key:
filter = True
value = int(stats[key])
if not filter:
if value != compare_value:
if "eq_dcpq:mapreduce_view" in key:
if value >= flow_control_buffer_size:
bucketMap[bucket] = False
else:
bucketMap[bucket] = False
return bucketMap
class MemcachedHelper(object):
def __init__(self, serverip = "localhost", port = 11211, bucket = "default", password = ""):
self.client = MemcachedClient(serverip, port)
self.client.sasl_auth_plain(bucket, password)
def write_one_json(self, key, doc):
count = 0
loaded = False
while count < 60 and not loaded:
try:
#self.client.set(key, 0, 0, json.dumps(doc))
self.client.set(key, 0, 0, doc)
loaded = True
except MemcachedError as error:
if error.status == 134:
print "Memcached TMP_OOM, Retrying in 5 seconds..."
count += 1
time.sleep(5)
elif error.status == 7:
print "Not my vbucket error. If on MAC, please specify vbuckets as 64."
print "If rebalance is in progress. Please wait for it to finish.\n"
break
else:
print error
break
def read_one_json(self, key):
doc = ""
try:
_, _, doc = self.client.get(key)
except MemcachedError as error:
print error
return doc
def write_batch(self, batch_id, data):
print "Batch Insert not implemented for Memcached"
def _add_conn(self, server):
if not self.store:
print("<%s> failed to add conn, invalid store object"\
% self.__class__.__name__)
return False
if self.store.__class__.__name__ == "StoreMembaseBinary":
print("<%s> _add_conn: %s"\
% (self.__class__.__name__, server))
host, port = server.split(":")
conn = MemcachedClient(host, int(port))
self.conns[server] = conn
return True
def _build_conns(self):
"""build separate connections based on store"""
if not self.store:
print("<%s> failed to build connections, invalid store object"\
% self.__class__.__name__)
return False
if self.store.__class__.__name__ == "StoreMemcachedBinary":
conn = MemcachedClient(self.store.conn.host, self.store.conn.port)
server_str = "{0}:{1}".format(self.store.conn.host,
self.store.conn.port)
self.conns[server_str] = conn
elif self.store.__class__.__name__ == "StoreMembaseBinary":
for memcached in self.store.awareness.memcacheds.values():
conn = MemcachedClient(memcached.host, memcached.port)
server_str = "{0}:{1}".format(conn.host, conn.port)
self.conns[server_str] = conn
self.awareness = self.store.awareness
else:
print("<%s> error: unsupported store object %s" %\
(self.__class__.__name__, store.__class__.__name__))
return False
return True
def __init__(self, serverip = "localhost", port = 11211, bucket = "default", password = ""):
self.client = MemcachedClient(serverip, port)
self.client.sasl_auth_plain(bucket, password)
def test_xattr_compression(self):
# MB-32669
# subdoc.subdoc_simple_dataset.SubdocSimpleDataset.test_xattr_compression,compression=active
mc = MemcachedClient(self.master.ip, 11210)
mc.sasl_auth_plain(self.master.rest_username,
self.master.rest_password)
mc.bucket_select('default')
self.key = "test_xattr_compression"
self.nesting_level = 5
array = {'i_add': 0, 'i_sub': 1, 'a_i_a': [0, 1], 'ai_sub': [0, 1]}
base_json = self.generate_json_for_nesting()
nested_json = self.generate_nested(base_json, array,
self.nesting_level)
jsonDump = json.dumps(nested_json)
stats = mc.stats()
self.assertEquals(stats['ep_compression_mode'], 'active')
scheme = "http"
host = "{0}:{1}".format(self.master.ip, self.master.port)
self.sdk_client = SDKClient(scheme=scheme,
hosts=[host],
bucket="default")
self.sdk_client.set(self.key, value=jsonDump, ttl=60)
rv = self.sdk_client.cb.mutate_in(self.key,
SD.upsert('my.attr',
"value",
xattr=True,
create_parents=True),
ttl=60)
self.assertTrue(rv.success)
# wait for it to persist and then evict the key
persisted = 0
while persisted == 0:
opaque, rep_time, persist_time, persisted, cas = mc.observe(
self.key)
mc.evict_key(self.key)
time.sleep(65)
try:
self.client.get(self.key)
self.fail("the key should get expired")
except mc_bin_client.MemcachedError as error:
self.assertEquals(error.status, 1)
stats = mc.stats()
self.assertEquals(int(stats['curr_items']), 0)
self.assertEquals(int(stats['curr_temp_items']), 0)
