def test_cbas_ingestion_with_documents_containing_multilingual_data(self): """ 1. Create reference to SDK client 2. Add multilingual json documents to default bucket 3. Verify ingestion on dataset with and with out secondary index """ multilingual_strings = [ 'De flesta sagorna här är från Hans Hörner svenska översättning', 'Il était une fois une maman cochon qui avait trois petits cochons', '森林里住着一只小兔子,它叫“丑丑”。它的眼睛红红的,像一对红宝石', '外治オヒル回条フ聞定ッ加官言岸ムモヱツ求碁込ヌトホヒ舞高メ旅位', 'ان عدة الشهور عند الله اثنا عشر شهرا في', ] self.log.info("Fetch test case arguments") self.fetch_test_case_arguments() self.log.info("Create reference to SDK client") client = SDKClient(scheme="couchbase", hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) self.log.info("Add multilingual documents to the default bucket") client.insert_custom_json_documents("custom-key-", multilingual_strings) self.log.info("Create connections, datasets and indexes") self.cbas_dataset_setup() self.log.info("Wait for ingestion to complete and verify count") self.cbas_util.wait_for_ingestion_complete([self.dataset_name], len(multilingual_strings)) self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count(self.dataset_name, len(multilingual_strings)))
def test_ingestion_impact_for_documents_containing_xattr_meta_information(self): self.log.info("Fetch test case arguments") self.fetch_test_case_arguments() self.log.info("Create reference to SDK client") client = SDKClient(scheme="couchbase", hosts=[self.master.ip], bucket="default", password=self.master.rest_password) self.log.info("Insert custom data into default bucket") documents = ['{"name":"value"}'] * self.num_of_documents document_id_prefix = "id-" client.insert_custom_json_documents(document_id_prefix, documents) self.log.info("Create connections, datasets, indexes") self.cbas_dataset_setup() self.log.info("Wait for ingestion to complete and verify count") self.cbas_util.wait_for_ingestion_complete([self.dataset_name], self.num_of_documents) self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count(self.dataset_name, self.num_of_documents)) self.log.info("Insert xattr attribute for all the documents and assert document count on dataset") for i in range(self.num_of_documents): client.insert_xattr_attribute(document_id=document_id_prefix + str(i), path="a", value="{'xattr-value': 1}", xattr=True, create_parents=True) self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count(self.dataset_name, self.num_of_documents)) self.log.info("Update xattr attribute and assert document count on dataset") for i in range(self.num_of_documents): client.update_xattr_attribute(document_id=document_id_prefix + str(i), path="a", value="{'xattr-value': 11}", xattr=True, create_parents=True) self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count(self.dataset_name, self.num_of_documents))
def test_swap_rebalance_cb_cbas_together(self): self.log.info("Creates cbas buckets and dataset") wait_for_rebalance = self.input.param("wait_for_rebalance", True) dataset_count_query = "select count(*) from {0};".format( self.cbas_dataset_name) self.setup_for_test() self.log.info("Add KV node and don't rebalance") self.add_node(node=self.rebalanceServers[1], rebalance=False) self.log.info("Add cbas node and don't rebalance") self.add_node(node=self.rebalanceServers[3], rebalance=False) otpnodes = [] nodes = self.rest.node_statuses() for node in nodes: if node.ip == self.rebalanceServers[ 0].ip or node.ip == self.rebalanceServers[2].ip: otpnodes.append(node) self.log.info("Remove master node") self.remove_node(otpnode=otpnodes, wait_for_rebalance=wait_for_rebalance) self.master = self.rebalanceServers[1] self.log.info("Create instances pointing to new master nodes") c_utils = cbas_utils(self.rebalanceServers[1], self.rebalanceServers[3]) c_utils.createConn(self.cb_bucket_name) self.log.info("Create reference to SDK client") client = SDKClient(scheme="couchbase", hosts=[self.rebalanceServers[1].ip], bucket=self.cb_bucket_name, password=self.rebalanceServers[1].rest_password) self.log.info("Add more document to default bucket") documents = ['{"name":"value"}'] * (self.num_items // 10) document_id_prefix = "custom-id-" client.insert_custom_json_documents(document_id_prefix, documents) self.log.info( "Run queries as rebalance is in progress : Rebalance state:%s" % self.rest._rebalance_progress_status()) handles = c_utils._run_concurrent_queries( dataset_count_query, "immediate", 2000, batch_size=self.concurrent_batch_size) self.log.info("Log concurrent query status") self.cbas_util.log_concurrent_query_outcome(self.master, handles) if not c_utils.validate_cbas_dataset_items_count( self.cbas_dataset_name, self.num_items + (self.num_items // 10), 0): self.fail( "No. of items in CBAS dataset do not match that in the CB bucket" )
def verify_infer_schema_on_documents_with_overlapping_types_same_keys(self): self.log.info('Create unique documents') client = SDKClient(hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) documents = [ '{ "array": 20, "integer": 300.34345, "float":false, "boolean":["a", "b", "c"], "null":null, "string":"300"}' '{ "array": 300, "integer": ["a", "b", "c"], "float":1012.56756, "boolean":300, "null":null, "string":10345665}' '{ "array": [1, 2, 3], "integer": 10345665, "float":"Steve", "boolean":[[1, 2], [3, 4], []], "null":null, "string":20.343}' '{ "array": 20.343, "integer": [[1, 2], [3, 4], []], "float":0.00011, "boolean":[1, 1.1, 1.0, 0], "null":null, "string":[1, 1.1, 1.0, 0]}' '{ "array": 10345665, "integer": 1012.56756, "float":1012.56756, "boolean":false, "null":null, "string":[1, "hello", ["a", 1], 2.22]}' '{ "array": "Steve", "integer": 1, "float":[1, 2, 3], "boolean":false, "null":null, "string":1}' '{ "array": true, "integer": 1.1, "float":20.343, "boolean":["a", "b", "c"], "null":null, "string":300}' '{ "array": true, "integer": "Alex", "float":10345665, "boolean":1.1, "null":null, "string":0.00011}' '{ "array": 20.343, "integer": 20.343, "float":300, "boolean":true, "null":null, "string":true}' '{ "array": 1, "integer": 10345665, "float":300.34345, "boolean":1, "null":null, "string":true}' ] for index, document in enumerate(documents): client.insert_document(str(index), document) self.log.info('Create primary index on %s' % self.cb_bucket_name) self.rest.query_tool('CREATE PRIMARY INDEX idx on %s' % self.cb_bucket_name) self.log.info('Create dataset') self.cbas_util.createConn(self.cb_bucket_name) self.cbas_util.create_dataset_on_bucket(self.cb_bucket_name, self.cbas_dataset_name) self.log.info('Connect link') self.cbas_util.connect_link() self.log.info('Verify dataset count') self.cbas_util.validate_cbas_dataset_items_count(self.cbas_dataset_name, len(documents)) self.validate_infer_schema_response(len(documents))
def __load_chain(self, start_num=0): for i, cluster in enumerate(self.get_cb_clusters()): if self._rdirection == REPLICATION_DIRECTION.BIDIRECTION: if i > len(self.get_cb_clusters()) - 1: break else: if i >= len(self.get_cb_clusters()) - 1: break if not self._dgm_run: for bucket in cluster.get_buckets(): client = SDKClient(scheme="couchbase", hosts=[cluster.get_master_node().ip], bucket=bucket.name).cb for i in range(start_num, start_num + self._num_items): key = 'k_%s_%s' % (i, str(cluster).replace(' ', '_'). replace('.', '_').replace(',', '_').replace(':', '_')) value = {'xattr_%s' % i:'value%s' % i} client.upsert(key, value) client.mutate_in(key, SD.upsert('xattr_%s' % i, 'value%s' % i, xattr=True, create_parents=True)) partition = bucket.kvs[1].acquire_partition(key)#["partition"] if self.only_store_hash: value = str(crc32.crc32_hash(value)) res = client.get(key) partition.set(key, json.dumps(value), 0, res.flags) bucket.kvs[1].release_partition(key) else: cluster.load_all_buckets_till_dgm( active_resident_threshold=self._active_resident_threshold, items=self._num_items)
def test_logical_clock_ticks(self): self.log.info('starting test_logical_clock_ticks') payload = "name={0}&roles=admin&password=password".format( self.buckets[0].name) self.rest.add_set_builtin_user(self.buckets[0].name, payload) sdk_client = SDKClient(scheme='couchbase', hosts=[self.servers[0].ip], bucket=self.buckets[0].name) mc_client = MemcachedClientHelper.direct_client( self.servers[0], self.buckets[0]) shell = RemoteMachineShellConnection(self.servers[0]) # do a bunch of mutations to set the max cas gen_load = BlobGenerator('key-for-cas-test-logical-ticks', 'value-for-cas-test-', self.value_size, end=10000) self._load_all_buckets(self.master, gen_load, "create", 0) vbucket_stats = mc_client.stats('vbucket-details') base_total_logical_clock_ticks = 0 for i in range(self.vbuckets): #print vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'] base_total_logical_clock_ticks = base_total_logical_clock_ticks + int( vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']) self.log.info('The base total logical clock ticks is {0}'.format( base_total_logical_clock_ticks)) # move the system clock back so the logical counter part of HLC is used and the logical clock ticks # stat is incremented self.assertTrue( shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS), 'Failed to advance the clock') # do more mutations NUMBER_OF_MUTATIONS = 10000 gen_load = BlobGenerator('key-for-cas-test-logical-ticks', 'value-for-cas-test-', self.value_size, end=NUMBER_OF_MUTATIONS) self._load_all_buckets(self.master, gen_load, "create", 0) vbucket_stats = mc_client.stats('vbucket-details') time.sleep(30) total_logical_clock_ticks = 0 for i in range(self.vbuckets): total_logical_clock_ticks = total_logical_clock_ticks + int( vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']) self.log.info('The total logical clock ticks is {0}'.format( total_logical_clock_ticks)) self.assertTrue( total_logical_clock_ticks - base_total_logical_clock_ticks == NUMBER_OF_MUTATIONS, 'Expected clock tick {0} actual {1}'.format( NUMBER_OF_MUTATIONS, total_logical_clock_ticks - base_total_logical_clock_ticks))
def test_add_concurrent(self): DOCID = 'subdoc_doc_id' SERVER_IP = self.servers[0].ip ITERATIONS = 200 THREADS = 20 main_bucket = SDKClient(scheme="couchbase", hosts=[self.servers[0].ip], bucket='default').cb main_bucket.upsert(DOCID, {'recs':[]}) class Runner(Thread): def run(self, *args, **kw): cb = SDKClient(scheme="couchbase", hosts=[SERVER_IP], bucket='default').cb for x in range(ITERATIONS): cb.mutate_in(DOCID, SD.array_append('recs', 1)) thrs = [Runner() for x in range(THREADS)] [t.start() for t in thrs] [t.join() for t in thrs] obj = main_bucket.get(DOCID) array_entry_count = len(obj.value['recs']) self.assertTrue(array_entry_count == ITERATIONS * THREADS, 'Incorrect number of array entries. Expected {0} actual {1}'.format(ITERATIONS * THREADS, array_entry_count))
def verify_infer_schema_on_same_type_missing_fields(self): self.log.info('Create unique documents') client = SDKClient(hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) documents = [ '{ "array": [1,2,3,4,5,6], "integer": 10, "float":10.12, "boolean":true, "null":null, "object":{"array": [1,2,3,4,5,6], "integer": 10, "float":10.12, "boolean":true, "null":null}}', '{ "array": null, "integer": 10, "float":10.12, "boolean":true, "null":null, "object":{"array": [1,2,3,4,5,6], "integer": 10, "float":10.12, "boolean":true, "null":"null value"}}', '{ "array": [1,2,3,4,5,6], "integer": 10, "float":10.12, "boolean":true, "null":null, "object":{"array": null, "integer": null, "float":null, "boolean":null, "null":null}}' ] for index, document in enumerate(documents): client.insert_document(str(index), document) self.log.info('Create primary index on %s' % self.cb_bucket_name) self.rest.query_tool('CREATE PRIMARY INDEX idx on %s' % self.cb_bucket_name) self.log.info('Create dataset') self.cbas_util.createConn(self.cb_bucket_name) self.cbas_util.create_dataset_on_bucket(self.cb_bucket_name, self.cbas_dataset_name) self.log.info('Connect link') self.cbas_util.connect_link() self.log.info('Verify dataset count') self.cbas_util.validate_cbas_dataset_items_count(self.cbas_dataset_name, len(documents)) self.validate_infer_schema_response(len(documents))
def verify_infer_schema_on_unique_nested_documents(self): self.log.info('Create unique documents') client = SDKClient(hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) documents = [ '{"from": "user_512", "to": "user_768", "text": "Hey, that Beer you recommended is pretty fab, thx!", "sent_timestamp":476560}', '{"user_id": 512, "name": "Bob Likington", "email": "*****@*****.**", "sign_up_timestamp": 1224612317, "last_login_timestamp": 1245613101}', '{"user_id": 768, "name": "Simon Neal", "email": "*****@*****.**", "sign_up_timestamp": 1225554317, "last_login_timestamp": 1234166701, "country": "Scotland", "pro_account": true, "friends": [512, 666, 742, 1111]}', '{"photo_id": "ccbcdeadbeefacee", "size": { "w": 500, "h": 320, "unit": "px" }, "exposure": "1/1082", "aperture": "f/2.4", "flash": false, "camera": { "name": "iPhone 4S", "manufacturer": {"Company":"Apple", "Location": {"City":"California", "Country":"USA"} } }, "user_id": 512, "timestamp": [2011, 12, 13, 16, 31, 7]}' ] for index, document in enumerate(documents): client.insert_document(str(index), document) self.log.info('Create primary index on %s' % self.cb_bucket_name) self.rest.query_tool('CREATE PRIMARY INDEX idx on %s' % self.cb_bucket_name) self.log.info('Create dataset') self.cbas_util.createConn(self.cb_bucket_name) self.cbas_util.create_dataset_on_bucket(self.cb_bucket_name, self.cbas_dataset_name) self.log.info('Connect link') self.cbas_util.connect_link() self.log.info('Verify dataset count') self.cbas_util.validate_cbas_dataset_items_count(self.cbas_dataset_name, len(documents)) self.validate_infer_schema_response(len(documents), compare_with_n1ql=False)
def _direct_client(self, server, bucket, timeout=30): # CREATE SDK CLIENT if self.use_sdk_client: try: from sdk_client import SDKClient scheme = "couchbase" host = self.master.ip if self.master.ip == "127.0.0.1": scheme = "http" host="{0}:{1}".format(self.master.ip, self.master.port) return SDKClient(scheme=scheme, hosts = [host], bucket = bucket) except ImportError: from sdk_client3 import SDKClient return SDKClient(RestConnection(self.master), bucket = bucket) except Exception as ex: self.log.error("cannot load sdk client due to error {0}".format(str(ex))) # USE MC BIN CLIENT WHEN NOT USING SDK CLIENT return self.direct_mc_bin_client(server, bucket, timeout= timeout)
def test_MB_32114(self): try: from sdk_client import SDKClient except: from sdk_client3 import SDKClient import couchbase.subdocument as SD rest = RestConnection(self.master) client = VBucketAwareMemcached(rest, 'default') if self.maxttl: self._expiry_pager(self.master) sdk_client = SDKClient(scheme='couchbase', hosts=[self.master.ip], bucket='default') KEY_NAME = 'key1' for i in range(1000): mcd = client.memcached(KEY_NAME + str(i)) rc = mcd.set(KEY_NAME + str(i), 0, 0, json.dumps({'value': 'value2'})) sdk_client.mutate_in( KEY_NAME + str(i), SD.upsert("subdoc_key", "subdoc_val", xattr=True, create_parents=True)) # wait for it to persist persisted = 0 while persisted == 0: opaque, rep_time, persist_time, persisted, cas = client.observe( KEY_NAME + str(i)) start_time = time.time() self._load_doc_data_all_buckets(batch_size=1000) end_time = time.time() for i in range(1000): try: mcd = client.memcached(KEY_NAME + str(i)) _, flags, exp, seqno, cas = client.memcached( KEY_NAME + str(i)).getMeta(KEY_NAME + str(i)) rc = mcd.del_with_meta(KEY_NAME + str(i), 0, 0, 2, cas + 1) except MemcachedError as exp: self.fail("Exception with del_with meta - {0}".format(exp)) self.cluster.compact_bucket(self.master, "default") if self.maxttl: time_to_sleep = (self.maxttl - (end_time - start_time)) + 20 self.sleep(int(time_to_sleep)) else: self.sleep(60) active_bucket_items = rest.get_active_key_count("default") replica_bucket_items = rest.get_replica_key_count("default") print('active_bucket_items ', active_bucket_items) print('replica_bucket_items ', replica_bucket_items) if active_bucket_items * self.num_replicas != replica_bucket_items: self.fail("Mismatch in data !!!")
def test_sdk_client(self): """ Test SDK Client Calls """ scheme = "couchbase" host = self.master.ip if self.master.ip == "127.0.0.1": scheme = "http" host = "{0}:{1}".format(self.master.ip, self.master.port) client = SDKClient(scheme=scheme, hosts=[host], bucket="default") client.remove("1", quiet=True) client.insert("1", "{1:2}") flag, cas, val = client.get("1") self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas)) client.upsert("1", "{1:3}") client.touch("1", ttl=100) flag, cas, val = client.get("1") self.assertTrue(val == "{1:3}", val) self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas)) client.remove("1", cas=cas) client.incr("key", delta=20, initial=5) flag, cas, val = client.get("key") self.assertTrue(val == 5) self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas)) client.incr("key", delta=20, initial=5) flag, cas, val = client.get("key") self.assertTrue(val == 25) self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas)) client.decr("key", delta=20, initial=5) flag, cas, val = client.get("key") self.assertTrue(val == 5) print flag, cas, val client.upsert("key1", "document1") client.upsert("key2", "document2") client.upsert("key3", "document3") set = client.get_multi(["key1", "key2"]) self.log.info(set) client.upsert_multi({"key1": "{1:2}", "key2": "{3:2}"}) set = client.get_multi(["key1", "key2"]) self.log.info(set) client.touch_multi(["key1", "key2"], ttl=200) set = client.get_multi(["key1", "key2"]) self.log.info(set) data = client.observe("key1") self.log.info(data) data = client.observe_multi(["key1", "key2"]) self.log.info(data) stats = client.stats(["key1"]) self.log.info(stats) client.n1ql_request( client.n1ql_query('create primary index on default')).execute() query = client.n1ql_query('select * from default') request = client.n1ql_request(query) obj = request.get_single_result()._jsobj self.log.info(obj) client.close()
def test_ephemeral_bucket_NRU_eviction_access_in_the_delete_range(self): """ generate_load = BlobGenerator(EvictionKV.KEY_ROOT, 'param2', self.value_size, start=0, end=self.num_items) self._load_all_ephemeral_buckets_until_no_more_memory(self.servers[0], generate_load, "create", 0, self.num_items) # figure out how many items were loaded and load 10% more rest = RestConnection(self.servers[0]) itemCount = rest.get_bucket(self.buckets[0]).stats.itemCount self.log.info( 'Reached OOM, the number of items is {0}'.format( itemCount)) """ # select some keys which we expect to be adjacent to the kvs which will be deleted # and how many KVs should we select, maybe that is a parameter itemCount = 50000 max_delete_value = itemCount / 10 NUM_OF_ACCESSES = 50 keys_to_access = set() for i in range(NUM_OF_ACCESSES): keys_to_access.add(random.randint(0, max_delete_value)) # and then do accesses on the key set client = SDKClient(hosts=[self.master.ip], bucket=self.buckets[0]) for i in keys_to_access: # and we may want to parameterize the get at some point rc = client.get(EvictionKV.KEY_ROOT + str(i), no_format=True) # and then do puts to delete out stuff PERCENTAGE_TO_ADD = 10 incremental_kv_population = BlobGenerator(EvictionKV.KEY_ROOT, 'param2', self.value_size, start=itemCount, end=itemCount * PERCENTAGE_TO_ADD / 100) self._load_bucket(self.buckets[0], self.master, incremental_kv_population, "create", exp=0, kv_store=1) # and verify that the touched kvs are still there for i in keys_to_access: # and we may want to parameterize the get at some point rc = client.get(EvictionKV.KEY_ROOT + str(i), no_format=True) self.assertFalse( rc is None, 'Key {0} was incorrectly deleted'.format(EvictionKV.KEY_ROOT + str(i)))
def async_load_data_till_upgrade_completes(): self.log.info("Started doc operations on KV") client = SDKClient(hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) i = 0 while not self._STOP_INGESTION: client.insert_document( "key-id" + str(i), '{"name":"James_' + str(i) + '", "profession":"Pilot"}') i += 1
def async_load_data(): self.log.info( "Performing doc operations on KV until upgrade is in progress") client = SDKClient(hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) i = 0 while not _STOP_INGESTION: client.insert_document( "key-id" + str(i), '{"name":"James_' + str(i) + '", "profession":"Pilot"}') i += 1
def load_docs(self, node, num_docs, bucket='default', password='', exp=0, flags=0): host = node.ip + ":" + node.port client = SDKClient(bucket="default", hosts=[host], scheme="http") for i in range(num_docs): key = "key%s" % i rc = client.upsert(key, "value")
def _direct_client(self, server, bucket, timeout=30): # CREATE SDK CLIENT if self.use_sdk_client: try: from sdk_client import SDKClient scheme = "couchbase" host = self.master.ip if self.master.ip == "127.0.0.1": scheme = "http" host="{0}:{1}".format(self.master.ip,self.master.port) return SDKClient(scheme=scheme,hosts = [host], bucket = bucket, password=self.master.rest_password) except Exception, ex: self.log.error("cannot load sdk client due to error {0}".format(str(ex)))
def test_xattr_compression(self): # MB-32669 # subdoc.subdoc_simple_dataset.SubdocSimpleDataset.test_xattr_compression,compression=active mc = MemcachedClient(self.master.ip, 11210) mc.sasl_auth_plain(self.master.rest_username, self.master.rest_password) mc.bucket_select('default') self.key = "test_xattr_compression" self.nesting_level = 5 array = {'i_add': 0, 'i_sub': 1, 'a_i_a': [0, 1], 'ai_sub': [0, 1]} base_json = self.generate_json_for_nesting() nested_json = self.generate_nested(base_json, array, self.nesting_level) jsonDump = json.dumps(nested_json) stats = mc.stats() self.assertEquals(stats['ep_compression_mode'], 'active') scheme = "http" host = "{0}:{1}".format(self.master.ip, self.master.port) self.sdk_client = SDKClient(scheme=scheme, hosts=[host], bucket="default") self.sdk_client.set(self.key, value=jsonDump, ttl=60) rv = self.sdk_client.cb.mutate_in(self.key, SD.upsert('my.attr', "value", xattr=True, create_parents=True), ttl=60) self.assertTrue(rv.success) # wait for it to persist and then evict the key persisted = 0 while persisted == 0: opaque, rep_time, persist_time, persisted, cas = mc.observe( self.key) mc.evict_key(self.key) time.sleep(65) try: self.client.get(self.key) self.fail("the key should get expired") except mc_bin_client.MemcachedError as error: self.assertEquals(error.status, 1) stats = mc.stats() self.assertEquals(int(stats['curr_items']), 0) self.assertEquals(int(stats['curr_temp_items']), 0)
def setUp(self): super(CBASLimitPushdown, self).setUp() self.log.info("Create a reference to SDK client") client = SDKClient(hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) self.log.info("Insert documents in KV bucket") documents = [ '{"name":"dave","age":19,"gender":"Male","salary":50.0, "married":false, "employed":""}', '{"name":"evan","age":25,"gender":"Female","salary":100.15, "married":true}', '{"name":"john","age":44,"gender":"Male","salary":150.55, "married":null}', '{"name": "sara", "age": 20, "gender": "Female", "salary": 200.34, "married":false}', '{"name":"tom","age":31,"gender":"Male","salary":250.99, "married":true}' ] client.insert_json_documents("id-", documents) self.log.info("Create connection") self.cbas_util.createConn(self.cb_bucket_name) self.log.info("Create primary index on KV bucket") self.rest.query_tool("create primary index pri_idx on default") self.log.info("Create dataset") self.cbas_util.create_dataset_on_bucket(self.cb_bucket_name, self.cbas_dataset_name) self.log.info("Connect to Local link") self.cbas_util.connect_link() self.log.info("Wait for ingestion to complete") self.total_documents = self.rest.query_tool( CBASLimitQueries.BUCKET_COUNT_QUERY)['results'][0]['$1'] self.cbas_util.wait_for_ingestion_complete([self.cbas_dataset_name], self.total_documents) self.log.info("Fetch partitions") shell = RemoteMachineShellConnection(self.cbas_node) response = self.cbas_util.fetch_analytics_cluster_response(shell) if 'partitions' in response: self.partitions = len(response['partitions']) self.log.info("Number of data partitions on cluster %d" % self.partitions) else: self.fail( msg= "Partitions not found. Failing early to avoid unexpected results" )
def direct_client(self, server, bucket, timeout=30): # CREATE SDK CLIENT if self.use_sdk_client: self.log.info("--> Using SDK client") self.is_secure = TestInputSingleton.input.param("is_secure", False) try: from sdk_client import SDKClient self.log.info("--> Try to use SDK2 client") if not self.is_secure: scheme = "couchbase" else: scheme = "couchbases" host = self.master.ip if self.master.ip == "127.0.0.1": if not self.is_secure: scheme = "http" else: scheme = "https" host = "{0}:{1}".format(self.master.ip, self.master.port) return SDKClient(scheme=scheme, hosts=[host], bucket=bucket.name, username=server.rest_username, password=server.rest_password) except ImportError: self.log.info("--> Using SDK3 client") from sdk_client3 import SDKClient return SDKClient(RestConnection(self.master), bucket=bucket.name) except Exception as ex: self.log.error( "cannot load sdk client due to error {0}".format(str(ex))) # USE MC BIN CLIENT WHEN NOT USING SDK CLIENT self.log.info("--> Using Direct Memcached client") return self.direct_mc_bin_client(server, bucket, timeout=timeout)
def test_ephemeral_bucket_NRU_eviction(self): generate_load = BlobGenerator(EvictionKV.KEY_ROOT, 'param2', self.value_size, start=0, end=self.num_items) self._load_all_ephemeral_buckets_until_no_more_memory( self.servers[0], generate_load, "create", 0, self.num_items) # figure out how many items were loaded and load 10% more rest = RestConnection(self.servers[0]) itemCount = rest.get_bucket(self.buckets[0]).stats.itemCount self.log.info( 'Reached OOM, the number of items is {0}'.format(itemCount)) incremental_kv_population = BlobGenerator(EvictionKV.KEY_ROOT, 'param2', self.value_size, start=itemCount, end=itemCount * 1.1) self._load_bucket(self.buckets[0], self.master, incremental_kv_population, "create", exp=0, kv_store=1) # and then probe the keys that are left. For now print out a distribution but later apply some heuristic client = SDKClient(hosts=[self.master.ip], bucket=self.buckets[0]) NUMBER_OF_CHUNKS = 11 items_in_chunk = int(1.1 * itemCount / NUMBER_OF_CHUNKS) for i in range(NUMBER_OF_CHUNKS): keys_still_present = 0 for j in range(items_in_chunk): rc = client.get(EvictionKV.KEY_ROOT + str(i * items_in_chunk + j), no_format=True) if rc[2] is not None: keys_still_present = keys_still_present + 1 self.log.info( 'Chunk {0} has {1:.2f} percent items still present'.format( i, 100 * keys_still_present / (itemCount * 1.1 / NUMBER_OF_CHUNKS)))
def setUp(self): super(CbasStats, self).setUp() self.log.info("Add Json documents to default bucket") self.perform_doc_ops_in_all_cb_buckets(self.num_items, "create", 0, self.num_items) self.log.info("Create reference to SDK client") client = SDKClient(scheme="couchbase", hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) self.log.info("Insert binary data into default bucket") keys = ["%s" % (uuid.uuid4()) for i in range(0, self.num_items)] client.insert_binary_document(keys) self.log.info("Create connection") self.cbas_util.createConn(self.cb_bucket_name) self.log.info("Create bucket on CBAS") self.assertTrue( self.cbas_util.create_bucket_on_cbas( cbas_bucket_name=self.cbas_bucket_name, cb_bucket_name=self.cb_bucket_name, cb_server_ip=self.cb_server_ip), "bucket creation failed on cbas") self.log.info("Create dataset on the CBAS bucket") self.cbas_util.create_dataset_on_bucket( cbas_bucket_name=self.cb_bucket_name, cbas_dataset_name=self.cbas_dataset_name) self.log.info("Connect to Bucket") self.cbas_util.connect_to_bucket( cbas_bucket_name=self.cbas_bucket_name, cb_bucket_password=self.cb_bucket_password) statement = "select count(*) from {0} where mutated=0;".format( self.cbas_dataset_name) status, metrics, errors, results, response_handle = self.cbas_util.execute_statement_on_cbas_util( statement, mode=self.mode, timeout=75, analytics_timeout=120) if not self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, self.num_items): self.fail( "No. of items in CBAS dataset do not match that in the CB bucket" )
def test_nru_eviction_impact_on_cbas(self): self.log.info("Create dataset") self.cbas_util.create_dataset_on_bucket(self.cb_bucket_name, self.cbas_dataset_name) self.log.info("Connect to Local link") self.cbas_util.connect_link() self.log.info("Add documents until ram percentage") self.load_document_until_ram_percentage() self.log.info("Fetch current document count") bucket_helper = BucketHelper(self.master) item_count = bucket_helper.get_bucket( self.cb_bucket_name).stats.itemCount self.log.info("Completed base load with %s items" % item_count) self.log.info( "Fetch initial inserted 100 documents, so they are not removed") client = SDKClient(hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) for i in range(100): client.get("test_docs-" + str(i)) self.log.info("Add 20% more items to trigger NRU") for i in range(item_count, int(item_count * 1.2)): client.insert_document("key-id" + str(i), '{"name":"dave"}') self.log.info("Validate document count on CBAS") count_n1ql = self.rest.query_tool( 'select count(*) from %s' % (self.cb_bucket_name))['results'][0]['$1'] if self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, count_n1ql): pass else: self.log.info( "Document count mismatch might be due to ejection of documents on KV. Retry again" ) count_n1ql = self.rest.query_tool( 'select count(*) from %s' % (self.cb_bucket_name))['results'][0]['$1'] self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, count_n1ql), msg="Count mismatch on CBAS")
def test_no_eviction_impact_on_cbas(self): self.log.info("Create dataset") self.cbas_util.create_dataset_on_bucket(self.cb_bucket_name, self.cbas_dataset_name) self.log.info("Connect to Local link") self.cbas_util.connect_link() self.log.info("Add documents until ram percentage") self.load_document_until_ram_percentage() self.log.info("Fetch current document count") bucket_helper = BucketHelper(self.master) item_count = bucket_helper.get_bucket( self.cb_bucket_name).stats.itemCount self.log.info("Completed base load with %s items" % item_count) self.log.info("Load more until we are out of memory") client = SDKClient(hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) i = item_count insert_success = True while insert_success: insert_success = client.insert_document("key-id" + str(i), '{"name":"dave"}') i += 1 self.log.info('Memory is full at {0} items'.format(i)) self.log.info("As a result added more %s items" % (i - item_count)) self.log.info("Fetch item count") stats = bucket_helper.get_bucket(self.cb_bucket_name).stats itemCountWhenOOM = stats.itemCount memoryWhenOOM = stats.memUsed self.log.info('Item count when OOM {0} and memory used {1}'.format( itemCountWhenOOM, memoryWhenOOM)) self.log.info("Validate document count on CBAS") count_n1ql = self.rest.query_tool( 'select count(*) from %s' % (self.cb_bucket_name))['results'][0]['$1'] self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, count_n1ql), msg="Count mismatch on CBAS")
def test_error_response_type_mismatch_object(self): self.log.info("Create a reference to SDK client") client = SDKClient(hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) self.log.info("Insert documents in KV bucket") documents = ['{"address":{"city":"NY"}}'] client.insert_json_documents("id-", documents) self.log.info("Create dataset and connect link") self.create_dataset_connect_link() status, _, errors, _, _ = self.cbas_util.execute_statement_on_cbas_util( self.error_response["query"]) self.validate_error_response(status, errors, self.error_response["msg"], self.error_response["code"])
def test_MB_36087(self): try: from sdk_client import SDKClient except: from sdk_client3 import SDKClient import couchbase.subdocument as SD g_key = "test_doc" bucket_name = "default" sdk_client = SDKClient(scheme='couchbase', hosts=[self.master.ip], bucket=bucket_name) rest = RestConnection(self.master) client = VBucketAwareMemcached(rest, bucket_name) for i in range(self.num_items): key = g_key + str(i) mcd = client.memcached(key) rc = mcd.set(key, 0, 0, json.dumps({'value': 'value2'})) sdk_client.mutate_in( key, SD.upsert("subdoc_key", "subdoc_val", xattr=True, create_parents=True)) # Wait for key to persist persisted = 0 while persisted == 0: opaque, rep_time, persist_time, persisted, cas = \ client.observe(key) time.sleep(10) # Evict the key try: rc = mcd.evict_key(key) except MemcachedError as exp: self.fail("Exception with evict meta - %s" % exp) # Perform del_with_meta try: mcd = client.memcached(key) _, flags, exp, seqno, cas = client.memcached(key).getMeta(key) rc = mcd.del_with_meta(key, 0, 0, 2, cas + 1) except MemcachedError as exp: self.fail("Exception with del_with meta - {0}".format(exp))
def key_not_exists_test(self): client = SDKClient(hosts = [self.master.ip], bucket = "default") KEY_NAME = 'key' for i in range(1500): client.set(KEY_NAME, "x") #For some reason could not get delete to work client.remove(KEY_NAME) rc = client.get(KEY_NAME) #.get is automatically set to quiet for the sdk_client, therefore I look for #none to indicate an error, otherwise the sdk_client spends 10 seconds trying #to retry the commands and is very slow if rc[2] == None: pass else: assert False #cas errors do not sleep the test for 10 seconds, plus we need to check that the correct #error is being thrown try: #For some reason replace instead of cas would not reproduce the bug client.cas(KEY_NAME, "value", cas = 10) except NotFoundError: pass assert True
def test_sdk_subddoc(self): """ Test SDK Client Calls """ scheme = "couchbase" host = self.master.ip if self.master.ip == "127.0.0.1": scheme = "http" host = "{0}:{1}".format(self.master.ip, self.master.port) client = SDKClient(scheme=scheme, hosts=[host], bucket="default") json_document = {"1": 1, "2": 2, "array": [1]} document_key = "1" client.insert("1", json_document) client.insert_in(document_key, "3", 3) client.upsert_in(document_key, "4", 4) client.upsert_in(document_key, "4", "change_4") client.replace_in(document_key, "4", "crap_4") client.arrayprepend_in(document_key, "array", "0") client.arrayappend_in(document_key, "array", "2") client.arrayinsert_in(document_key, "array[1]", "INSERT_VALUE_AT_INDEX_1") client.arrayaddunique_in(document_key, "array", "INSERT_UNIQUE_VALUE") print json.dumps(client.get(document_key))
def test_cbas_bucket_connect_with_more_than_eight_active_datasets(self): """ 1. Create a cb bucket 2. Create a cbas bucket 3. Create 9 datasets 4. Connect to cbas bucket must fail with error - Maximum number of active writable datasets (8) exceeded 5. Delete 1 dataset, now the count must be 8 6. Re-connect the cbas bucket and this time connection must succeed 7. Verify count in dataset post connect """ self.log.info("Fetch test case arguments") self.fetch_test_case_arguments() self.log.info("Load data in the default bucket") self.perform_doc_ops_in_all_cb_buckets(self.num_items, "create", 0, self.num_items) self.log.info("Create reference to SDK client") client = SDKClient(scheme="couchbase", hosts=[self.master.ip], bucket=self.cb_bucket_name, password=self.master.rest_password) self.log.info("Insert binary data into default bucket") keys = ["%s" % (uuid.uuid4()) for i in range(0, self.num_items)] client.insert_binary_document(keys) self.log.info("Insert Non-Json string data into default bucket") keys = ["%s" % (uuid.uuid4()) for i in range(0, self.num_items)] client.insert_string_document(keys) self.log.info("Create connection") self.cbas_util.createConn(self.cb_bucket_name) self.log.info("Create a CBAS bucket") self.assertTrue(self.cbas_util.create_bucket_on_cbas( cbas_bucket_name=self.cbas_bucket_name, cb_bucket_name=self.cb_bucket_name), msg="Failed to create CBAS bucket") self.log.info("Create datasets") for i in range(1, self.num_of_dataset + 1): self.assertTrue( self.cbas_util.create_dataset_on_bucket( cbas_bucket_name=self.cb_bucket_name, cbas_dataset_name=self.dataset_prefix + str(i)), msg="Failed to create dataset {0}".format(self.dataset_prefix + str(i))) self.log.info("Verify connect to CBAS bucket must fail") self.assertTrue( self.cbas_util.connect_to_bucket( cbas_bucket_name=self.cbas_bucket_name, cb_bucket_password=self.cb_bucket_password, validate_error_msg=True, expected_error=BucketOperations.CBAS_BUCKET_CONNECT_ERROR_MSG), msg="Incorrect error msg while connecting to cbas bucket") self.log.info("Drop the last dataset created") self.assertTrue( self.cbas_util.drop_dataset(cbas_dataset_name=self.dataset_prefix + str(self.num_of_dataset)), msg="Failed to drop dataset {0}".format(self.dataset_prefix + str(self.num_of_dataset))) self.log.info("Connect to CBAS bucket") self.assertTrue(self.cbas_util.connect_to_bucket( cbas_bucket_name=self.cbas_bucket_name), msg="Failed to connect to cbas bucket") self.log.info("Wait for ingestion to complete and validate count") for i in range(1, self.num_of_dataset): self.cbas_util.wait_for_ingestion_complete( [self.dataset_prefix + str(i)], self.num_items) self.assertTrue( self.cbas_util.validate_cbas_dataset_items_count( self.dataset_prefix + str(i), self.num_items))
def run(self, *args, **kw): cb = SDKClient(scheme="couchbase", hosts=[SERVER_IP], bucket='default').cb for x in range(ITERATIONS): cb.mutate_in(DOCID, SD.array_append('recs', 1))