def _verify_data(self, version): #verify all the keys #let's use vbucketaware rest = RestConnection(self.servers[0]) moxi = MemcachedClientHelper.proxy_client(self.servers[0], self.bucket_name) index = 0 all_verified = True keys_failed = [] for key in self.updated_keys: try: index += 1 flag, keyx, value = moxi.get(key=key) self.assertTrue(value.endswith(version), msg='values do not match . key value should endwith {0}'.format(version)) except MemcachedError as error: self.log.error(error) self.log.error( "memcachedError : {0} - unable to get a pre-inserted key : {0}".format(error.status, key)) keys_failed.append(key) all_verified = False # except : # self.log.error("unknown errors unable to get a pre-inserted key : {0}".format(key)) # keys_failed.append(key) # all_verified = False self.assertTrue(all_verified, 'unable to verify #{0} keys'.format(len(keys_failed)))
def _dockey_data_ops_with_moxi(self, dockey="dockey", data_op="create"): expected_rows = self.num_items for bucket in self.buckets: try: client = MemcachedClientHelper.proxy_client( self.master, bucket.name) except Exception as ex: self.log.exception( "unable to create memcached client due to {0}..".format( ex)) try: for itr in range(self.num_items): key = dockey + str(itr) value = str(itr) if data_op in ["create", "update"]: client.set(key, 0, 0, value) elif data_op == "delete": client.delete(key) expected_rows = 0 except Exception as ex: self.log.exception( "Received exception {0} while performing data op - {1}". format(ex, data_op)) self._wait_for_stats_all_buckets(self.servers[:self.nodes_init]) if self.bucket_type != 'ephemeral': # views not supported for ephemeral buckets self._verify_with_views(expected_rows)
def data_ops_with_moxi(self, server, data_op, buckets, items, use_ascii): for bucket in buckets: try: client = MemcachedClientHelper.proxy_client( server, bucket.name, force_ascii=use_ascii) except Exception as ex: self.err = "unable to create memcached client due to {0}..".format( ex) try: for itr in xrange(items): key = 'bucketflush' + str(itr) value = 'bucketflush-' + str(itr) if data_op in ["create", "update"]: client.set(key, 0, 0, value) elif data_op == "delete": client.delete(key) except MemcachedError as exp: if exp.status != 134: self.log.info("Unexpected Exception Caught - {0}".format(exp)) self.err = "Expected Exception Caught - {0}".format(exp) except Exception as exp: self.log.info("Unexpected Exception Caught - {0}".format(exp)) self.err = "Unexpected exception caught- {0}".format(exp) else: self.err = "All buckets may not have been flushed"
def _insert_data(self, howmany): self.onenodemc = MemcachedClientHelper.proxy_client(self.master, "default") items = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, howmany)] for item in items: self.onenodemc.set(item, 0, 0, item) self.log.info("inserted {0} items".format(howmany)) self.onenodemc.close()
def test_MB_14288(self): mc = MemcachedClientHelper.proxy_client(self.master, "default") blob = bytearray(1024 * 1024 * 10) mc.set("MB-14288", 0, 0, blob) flags_v, cas_v, retrieved = mc.get("MB-14288") if not blob == retrieved: self.fail("It should be possible to store and retrieve values > 1M")
def insert_key(serverInfo, bucket_name, count, size): client = MemcachedClientHelper.proxy_client(serverInfo, bucket_name) value = MemcachedClientHelper.create_value("*", size) for i in range(count * 1000): key = "key_" + str(i) flag = random.randint(1, 999) client.set(key, 0, flag, value)
def incr_test(self, key, exp, flags, value, incr_amt, decr_amt, incr_time): global update_value serverInfo = self.master client = MemcachedClientHelper.proxy_client(serverInfo, self.bucket_name) # self.log.info('Waitting 15 seconds for memcached started') # time.sleep(15) if key != 'no_key': client.set(key, exp, flags, value) if exp: self.log.info('Wait {0} seconds for the key expired' .format(exp + 2)) time.sleep(exp + 2) if decr_amt: c, d = client.decr(key, decr_amt) self.log.info('decr amt {0}' .format(c)) try: i = 0 while i < incr_time: update_value, cas = client.incr(key, incr_amt) i += 1 self.log.info('incr {0} times with value {1}'.format(incr_time, incr_amt)) return update_value except mc_bin_client.MemcachedError as error: self.log.info('memcachedError : {0}'.format(error.status)) self.test.fail("unable to increment value: {0}".format(incr_amt))
def _verify_data(self, version): #verify all the keys #let's use vbucketaware moxi = MemcachedClientHelper.proxy_client(self.servers[0], self.bucket_name) index = 0 all_verified = True keys_failed = [] for key in self.updated_keys: try: index += 1 flag, keyx, value = moxi.get(key=key) self.assertTrue(value.endswith(version), msg='values do not match . key value should endwith {0}'.format(version)) except MemcachedError as error: self.log.error(error) self.log.error( "memcachedError : {0} - unable to get a pre-inserted key : {0}".format(error.status, key)) keys_failed.append(key) all_verified = False # except : # self.log.error("unknown errors unable to get a pre-inserted key : {0}".format(key)) # keys_failed.append(key) # all_verified = False self.assertTrue(all_verified, 'unable to verify #{0} keys'.format(len(keys_failed)))
def _insert_data(self, howmany): self.onenodemc = MemcachedClientHelper.proxy_client( self.master, "default") items = [ "{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, howmany) ] for item in items: self.onenodemc.set(item, 0, 0, item) self.log.info("inserted {0} items".format(howmany)) self.onenodemc.close()
def _insert_data(self, howmany): self.onenodemc = MemcachedClientHelper.proxy_client(self.master, "default") items = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, howmany)] try: for item in items: self.onenodemc.set(item, 0, 0, item) self.log.info("inserted {0} items".format(howmany)) self.onenodemc.close() except Exception as e: print e self.fail("could not set item to bucket")
def touch_test(self): timeout = 900 # 15 minutes stats_all_buckets = {} payload = MemcachedClientHelper.create_value('*', self.value_size) mc = MemcachedClientHelper.proxy_client(self.servers[0], "default") prefix = "test_" self.num_items = self.input.param("items", 10000) k = 0 while k < 100: key = "{0}{1}".format(prefix, k) mc.set(key, 0, 0, payload) k += 1 active_resident_threshold = 30 threshold_reached = False end_time = time.time() + float(timeout) while not threshold_reached and time.time() < end_time: if int(mc.stats()["vb_active_perc_mem_resident"] ) >= active_resident_threshold: self.log.info( "vb_active_perc_mem_resident_ratio reached at %s " % (mc.stats()["vb_active_perc_mem_resident"])) items = self.num_items self.num_items += self.input.param("items", 40000) random_key = self.key_generator() generate_load = BlobGenerator(random_key, '%s-' % random_key, self.value_size, end=self.num_items) self._load_all_buckets(self.servers[0], generate_load, "create", 0, True, batch_size=40000, pause_secs=3) else: threshold_reached = True self.log.info("DGM {0} state achieved!!!!".format( active_resident_threshold)) if time.time() > end_time and int(mc.stats( )["vb_active_perc_mem_resident"]) >= active_resident_threshold: raise Exception("failed to load items into bucket") """ at active resident ratio above, the first 100 keys insert into bucket will be non resident. Then do touch command to test it """ self.log.info( "Run touch command to test items which are in non resident.") k = 0 while k < 100: key = "{0}{1}".format(prefix, k) try: mc.touch(key, 0) k += 1 except Exception as ex: raise Exception(ex)
def keys_exist_or_assert(keys, server, bucket_name, test, queue=None, scope=None, collection=None): # we should try out at least three times log = logger.Logger.get_logger() # verify all the keys client = MemcachedClientHelper.proxy_client(server, bucket_name) # populate key retry = 1 keys_left_to_verify = [] keys_left_to_verify.extend(copy.deepcopy(keys)) log_count = 0 while retry < 6 and len(keys_left_to_verify) > 0: msg = "trying to verify {0} keys - attempt #{1} : {2} keys left to verify" log.info(msg.format(len(keys), retry, len(keys_left_to_verify))) keys_not_verified = [] for key in keys_left_to_verify: try: client.get(key=key, scope=scope, collection=collection) except mc_bin_client.MemcachedError as error: keys_not_verified.append(key) if log_count < 100: log.error("key {0} does not exist because {1}".format( key, error)) log_count += 1 retry += 1 keys_left_to_verify = keys_not_verified if len(keys_left_to_verify) > 0: log_count = 0 for key in keys_left_to_verify: log.error("key {0} not found".format(key)) log_count += 1 if log_count > 100: break msg = "unable to verify {0} keys".format(len(keys_left_to_verify)) log.error(msg) if test: queue.put(False) test.fail(msg=msg) if queue is None: return False else: queue.put(False) log.info("verified that {0} keys exist".format(len(keys))) if queue is None: return True else: queue.put(True)
def _insert_data_till_stopped(self, bucket, prefix, number_of_items): moxi = MemcachedClientHelper.proxy_client(self.servers[0], bucket) inserted_index = [] while not self.shutdown_load_data: for i in range(0, number_of_items): key = doc_name = "{0}-{1}".format(prefix, i) value = {"name": doc_name, "age": random.randint(0, 5000)} try: moxi.set(key, 0, 0, json.dumps(value)) inserted_index.append(i) except Exception as ex: self.log.error("unable to set item , error {0}".format(ex)) self.log.info("inserted {0} json documents".format(inserted_index))
def _insert_data_till_stopped(self, bucket, prefix, number_of_items): moxi = MemcachedClientHelper.proxy_client(self.servers[0], bucket) inserted_index = [] while not self.shutdown_load_data: for i in range(0, number_of_items): key = doc_name = "{0}-{1}".format(prefix, i) value = {"name": doc_name, "age": random.randint(0, 5000)} try: moxi.set(key, 0, 0, json.dumps(value)) inserted_index.append(i) except Exception as ex: self.log.error("unable to set item , error {0}".format(ex)) self.log.info("inserted {0} json documents".format(inserted_index))
def insert_docs(self, num_of_docs, prefix, extra_values={}, wait_for_persistence=True, return_docs=False): moxi = MemcachedClientHelper.proxy_client(self.master, self.bucket) doc_names = [] for i in range(0, num_of_docs): key = doc_name = "{0}-{1}".format(prefix, i) geom = { "type": "Point", "coordinates": [random.randrange(-180, 180), random.randrange(-90, 90)] } value = { "name": doc_name, "age": random.randrange(1, 1000), "geometry": geom } value.update(extra_values) if not return_docs: doc_names.append(doc_name) else: doc_names.append(value) # loop till value is set fail_count = 0 while True: try: moxi.set(key, 0, 0, json.dumps(value)) break except MemcachedError as e: fail_count += 1 if (e.status == 133 or e.status == 132) and fail_count < 60: if i == 0: self.log.error( "moxi not fully restarted, " "waiting 5 seconds. error {0}".format(e)) time.sleep(5) else: self.log.error(e) time.sleep(1) else: raise e if wait_for_persistence: self.wait_for_persistence() self.log.info("inserted {0} json documents".format(num_of_docs)) return doc_names
def _insert_data(self, howmany): self.onenodemc = MemcachedClientHelper.proxy_client( self.master, "default") items = [ "{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, howmany) ] try: for item in items: self.onenodemc.set(item, 0, 0, item) self.log.info("inserted {0} items".format(howmany)) self.onenodemc.close() except Exception as e: print(e) self.fail("could not set item to bucket")
def _eject_items(self, item_count, prefix): client = MemcachedClientHelper.proxy_client(self.master, self.default_bucket_name) time_start = time.time() for i in range(item_count): timeout_end = time.time() + 10 passed = False while time.time() < timeout_end and not passed: try: client.evict_key(prefix + "_key_" + str(i)) passed = True except Exception as e: self.log.error("failed to eject key {0}, error: {1}".format(prefix + "_key_" + str(i), e)) time.sleep(2) self.assertTrue(passed, "exit due to eject errors after {0} seconds".format(time.time() - time_start)) self.log.info("ejected {0} items in {1} seconds".format(item_count, time.time() - time_start))
def setUp(self): self.log = logger.Logger.get_logger() self.params = TestInputSingleton.input.test_params self.master = TestInputSingleton.input.servers[0] rest = RestConnection(self.master) rest.init_cluster(self.master.rest_username, self.master.rest_password) info = rest.get_nodes_self() rest.init_cluster_memoryQuota(self.master.rest_username, self.master.rest_password, memoryQuota=info.mcdMemoryReserved) ClusterOperationHelper.cleanup_cluster([self.master]) ClusterOperationHelper.wait_for_ns_servers_or_assert([self.master], self) self._create_default_bucket() self.keys_cleanup = [] self.onenodemc = MemcachedClientHelper.direct_client(self.master, "default", timeout=600) self.onenodemoxi = MemcachedClientHelper.proxy_client(self.master, "default", timeout=600)
def _load_items(self, item_count, expiration, prefix, vprefix=""): flags = 0 client = MemcachedClientHelper.proxy_client(self.master, self.default_bucket_name) time_start = time.time() for i in range(item_count): timeout_end = time.time() + 10 passed = False while time.time() < timeout_end and not passed: try: client.set(prefix + "_key_" + str(i), expiration, flags, vprefix + "_value_" + str(i)) passed = True except Exception as e: self.log.error("failed to set key {0}, error: {1}".format(prefix + "_key_" + str(i), e)) time.sleep(2) self.assertTrue(passed, "exit due to set errors after {0} seconds".format(time.time() - time_start)) self.log.info("loaded {0} items in {1} seconds".format(item_count, time.time() - time_start))
def set_test(self, key, exp, flags, values): serverInfo = self.master client = MemcachedClientHelper.proxy_client(serverInfo, self.bucket_name,) # self.log.info('Waitting 15 seconds for memcached started') # time.sleep(15) for v in values: for f in flags: client.set(key, exp, f, v) flags_v, cas_v, get_v = client.get(key) if get_v == v: if flags_v == f: self.log.info('Flags is set to {0}; and when run get {1}'.format(f, flags_v)) else: self.test.fail('FAILED. Flags is set to {0}; and when run get {1}'.format(f, flags_v)) self.log.info('Value is set {0}; and when run get {1}'.format(v, get_v)) else: self.test.fail('FAILED. Value is set to {0}; and when run get {1}'.format(v, get_v))
def keys_exist_or_assert(keys, server, bucket_name, test, queue=None): # we should try out at least three times log = logger.Logger.get_logger() # verify all the keys client = MemcachedClientHelper.proxy_client(server, bucket_name) # populate key retry = 1 keys_left_to_verify = [] keys_left_to_verify.extend(copy.deepcopy(keys)) log_count = 0 while retry < 6 and len(keys_left_to_verify) > 0: msg = "trying to verify {0} keys - attempt #{1} : {2} keys left to verify" log.info(msg.format(len(keys), retry, len(keys_left_to_verify))) keys_not_verified = [] for key in keys_left_to_verify: try: client.get(key=key) except mc_bin_client.MemcachedError as error: keys_not_verified.append(key) if log_count < 100: log.error("key {0} does not exist because {1}".format(key, error)) log_count += 1 retry += 1 keys_left_to_verify = keys_not_verified if len(keys_left_to_verify) > 0: log_count = 0 for key in keys_left_to_verify: log.error("key {0} not found".format(key)) log_count += 1 if log_count > 100: break msg = "unable to verify {0} keys".format(len(keys_left_to_verify)) log.error(msg) if test: queue.put(False) test.fail(msg=msg) if queue is None: return False else: queue.put(False) log.info("verified that {0} keys exist".format(len(keys))) if queue is None: return True else: queue.put(True)
def _update_keys(self, version): rejected_keys = [] #quit after updating max 100,000 keys self.updated_keys = [] moxi = MemcachedClientHelper.proxy_client(self.servers[0], self.bucket_name) for key in self.keys: if len(self.updated_keys) > 10000: break value = '{0}'.format(version) try: moxi.append(key, value) self.updated_keys.append(key) except MemcachedError: # self.log.error(error) # self.log.error("unable to update key : {0} to bucket : {1}".format(key, client.vbucketId)) rejected_keys.append(key) if len(rejected_keys) > 0: self.log.error("unable to update {0} keys".format(len(rejected_keys)))
def decr_test(self, key, exp, flags, value, incr_amt, decr_amt, decr_time): global update_value serverInfo = self.master client = MemcachedClientHelper.proxy_client(serverInfo, self.bucket_name) if key != 'no_key': client.set(key, exp, flags, value) if exp: self.log.info('Wait {0} seconds for the key expired' .format(exp + 2)) time.sleep(exp + 2) if incr_amt: c, d = client.incr(key, incr_amt) self.log.info('incr amt {0}' .format(c)) i = 0 while i < decr_time: update_value, cas = client.decr(key, decr_amt) i += 1 self.log.info('decr {0} times with value {1}'.format(decr_time, decr_amt)) return update_value
def _get_items(self, item_count, prefix, vprefix=""): client = MemcachedClientHelper.proxy_client(self.master, self.default_bucket_name) time_start = time.time() get_count = 0 last_error = "" error_count = 0 for i in range(item_count): try: value = client.get(prefix + "_key_" + str(i))[2] assert(value == vprefix + "_value_" + str(i)) get_count += 1 except Exception as e: last_error = "failed to getr key {0}, error: {1}".format(prefix + "_key_" + str(i), e) error_count += 1 if error_count > 0: self.log.error("got {0} errors, last error: {1}".format(error_count, last_error)) self.log.info("got {0} replica items in {1} seconds".format(get_count, time.time() - time_start)) return get_count
def _update_keys(self, version): rejected_keys = [] #quit after updating max 100,000 keys self.updated_keys = [] moxi = MemcachedClientHelper.proxy_client(self.servers[0], self.bucket_name) for key in self.keys: if len(self.updated_keys) > 10000: break value = '{0}'.format(version) try: moxi.append(key, value) self.updated_keys.append(key) except MemcachedError: # self.log.error(error) # self.log.error("unable to update key : {0} to bucket : {1}".format(key, client.vbucketId)) rejected_keys.append(key) if len(rejected_keys) > 0: self.log.error("unable to update {0} keys".format(len(rejected_keys)))
def delete_docs(self, num_of_docs, prefix): moxi = MemcachedClientHelper.proxy_client(self.master, self.bucket) doc_names = [] for i in range(0, num_of_docs): key = "{0}-{1}".format(prefix, i) try: moxi.delete(key) except MemcachedError as e: if e.args[0] == "Memcached error #1: Not found": continue else: raise doc_names.append(key) self.wait_for_persistence(180) self.log.info("deleted {0} json documents".format(len(doc_names))) return doc_names
def delete_docs(self, num_of_docs, prefix): moxi = MemcachedClientHelper.proxy_client(self.master, self.bucket) doc_names = [] for i in range(0, num_of_docs): key = "{0}-{1}".format(prefix, i) try: moxi.delete(key) except MemcachedError as e: # Don't care if we try to delete a document that doesn't exist if e.status == memcacheConstants.ERR_NOT_FOUND: continue else: raise doc_names.append(key) self.wait_for_persistence(180) self.log.info("deleted {0} json documents".format(len(doc_names))) return doc_names
def delete_docs(self, num_of_docs, prefix): moxi = MemcachedClientHelper.proxy_client(self.master, self.bucket) doc_names = [] for i in range(0, num_of_docs): key = "{0}-{1}".format(prefix, i) try: moxi.delete(key) except MemcachedError as e: # Don't care if we try to delete a document that doesn't exist if e.status == memcacheConstants.ERR_NOT_FOUND: continue else: raise doc_names.append(key) self.wait_for_persistence(180) self.log.info("deleted {0} json documents".format(len(doc_names))) return doc_names
def data_ops_with_moxi(self, server, data_op, buckets, items, use_ascii): for bucket in buckets: try: client = MemcachedClientHelper.proxy_client(server, bucket.name, force_ascii=use_ascii) except Exception as ex: self.log.error("unable to create memcached client due to {0}..".format(ex)) with self.assertRaises(MemcachedError) as exp: for itr in xrange(items): key = 'bucketflush' + str(itr) value = 'bucketflush-' + str(itr) if data_op in ["create", "update"]: client.set(key, 0, 0, value) elif data_op == "delete": client.delete(key) memcached_exception = exp.exception self.assertEqual(memcached_exception.status, 134, msg="Unexpected Exception - {0}".format(memcached_exception)) self.log.info("Expected Exception Caught - {0}".format(memcached_exception))
def data_ops_with_moxi(self, server, data_op, buckets, items, use_ascii): for bucket in buckets: try: client = MemcachedClientHelper.proxy_client(server, bucket.name, force_ascii=use_ascii) except Exception as ex: self.log.error("unable to create memcached client due to {0}..".format(ex)) with self.assertRaises(MemcachedError) as exp: for itr in xrange(items): key = "bucketflush" + str(itr) value = "bucketflush-" + str(itr) if data_op in ["create", "update"]: client.set(key, 0, 0, value) elif data_op == "delete": client.delete(key) memcached_exception = exp.exception self.assertEqual(memcached_exception.status, 134, msg="Unexpected Exception - {0}".format(memcached_exception)) self.log.info("Expected Exception Caught - {0}".format(memcached_exception))
def _reader_thread(self, inserted_keys, bucket_data, moxi=False): errors = [] rest = RestConnection(self._servers[0]) smartclient = None for name in bucket_data: for key in inserted_keys: if moxi: moxi = MemcachedClientHelper.proxy_client(self._servers[0], name) else: smartclient = VBucketAwareMemcached(rest, name) try: if moxi: moxi.get(key) else: smartclient.memcached(key).get(key) except Exception as ex: errors.append({"error": ex, "key": key}) self.log.info(ex) if not moxi: smartclient.done() smartclient = VBucketAwareMemcached(rest, name)
def _dockey_data_ops_with_moxi(self, dockey="dockey", data_op="create"): expected_rows = self.num_items for bucket in self.buckets: try: client = MemcachedClientHelper.proxy_client(self.master, bucket.name) except Exception as ex: self.log.exception("unable to create memcached client due to {0}..".format(ex)) try: for itr in xrange(self.num_items): key = dockey + str(itr) value = str(itr) if data_op in ["create", "update"]: client.set(key, 0, 0, value) elif data_op == "delete": client.delete(key) expected_rows = 0 except Exception as ex: self.log.exception("Received exception {0} while performing data op - {1}".format(ex, data_op)) self._wait_for_stats_all_buckets(self.servers[:self.nodes_init]) self._verify_with_views(expected_rows)
def touch_test(self): timeout = 900 # 15 minutes stats_all_buckets = {} payload = MemcachedClientHelper.create_value('*', self.value_size) mc = MemcachedClientHelper.proxy_client(self.servers[0], "default") prefix = "test_" self.num_items = self.input.param("items", 10000) k = 0 while k < 100: key = "{0}{1}".format(prefix, k) mc.set(key, 0, 0, payload) k += 1 active_resident_threshold = 30 threshold_reached = False end_time = time.time() + float(timeout) while not threshold_reached and time.time() < end_time: if int(mc.stats()["vb_active_perc_mem_resident"]) >= active_resident_threshold: self.log.info("vb_active_perc_mem_resident_ratio reached at %s " % (mc.stats()["vb_active_perc_mem_resident"])) items = self.num_items self.num_items += self.input.param("items", 40000) random_key = self.key_generator() generate_load = BlobGenerator(random_key, '%s-' % random_key, self.value_size, end=self.num_items) self._load_all_buckets(self.servers[0], generate_load, "create", 0, True, batch_size=40000, pause_secs=3) else: threshold_reached = True self.log.info("DGM {0} state achieved!!!!".format(active_resident_threshold)) if time.time() > end_time and int(mc.stats()["vb_active_perc_mem_resident"]) >= active_resident_threshold: raise Exception("failed to load items into bucket") """ at active resident ratio above, the first 100 keys insert into bucket will be non resident. Then do touch command to test it """ self.log.info("Run touch command to test items which are in non resident.") k = 0 while k < 100: key = "{0}{1}".format(prefix, k) try: mc.touch(key, 0) k += 1 except Exception as ex: raise Exception(ex)
def insert_docs(self, num_of_docs, prefix, extra_values={}, wait_for_persistence=True, return_docs=False): moxi = MemcachedClientHelper.proxy_client(self.master, self.bucket) doc_names = [] for i in range(0, num_of_docs): key = doc_name = "{0}-{1}".format(prefix, i) geom = {"type": "Point", "coordinates": [random.randrange(-180, 180), random.randrange(-90, 90)]} value = {"name": doc_name, "age": 1000, "geometry": geom} value.update(extra_values) if not return_docs: doc_names.append(doc_name) else: doc_names.append(value) # loop till value is set fail_count = 0 while True: try: moxi.set(key, 0, 0, json.dumps(value)) break except MemcachedError as e: fail_count += 1 if (e.status == 133 or e.status == 132) and fail_count < 60: if i == 0: self.log.error("moxi not fully restarted, " "waiting 5 seconds. error {0}" .format(e)) time.sleep(5) else: self.log.error(e) time.sleep(1) else: raise e if wait_for_persistence: self.wait_for_persistence() self.log.info("inserted {0} json documents".format(num_of_docs)) return doc_names
def data_ops_with_moxi(self, server, data_op, buckets, items, use_ascii): for bucket in buckets: try: client = MemcachedClientHelper.proxy_client(server, bucket.name, force_ascii=use_ascii) except Exception as ex: self.log.error("unable to create memcached client due to {0}..".format(ex)) try: for itr in xrange(items): key = 'bucketflush' + str(itr) value = 'bucketflush-' + str(itr) if data_op in ["create", "update"]: client.set(key, 0, 0, value) elif data_op == "delete": client.delete(key) except MemcachedError as exp: self.assertEqual(exp.status, 134, msg="Unexpected Exception - {0}".format(exp)) self.log.info("Expected Exception Caught - {0}".format(exp)) except Exception as exp: self.log.info("Unxpected Exception Caught - {0}".format(exp)) self.fail("Unexpected exception caught- {0}".format(exp)) else: self.fail("All buckets may not have been flushed")
expiry = 0 if "count" in params: count = int(params["count"]) print count if "size" in params: size = int(params["size"]) if "prefix" in params: prefix = params["prefix"] #if "expiry" in params: # expiry = int(params["expiry"]) payload = MemcachedClientHelper.create_value('*', size) rest = RestConnection(server) buckets = rest.get_buckets() for bucket in buckets: smart = VBucketAwareMemcached(rest, bucket.name) mc = MemcachedClientHelper.proxy_client(server, bucket.name) i = 0 counter_10 = 0 all_set = False while i < count: try: key = "{0}-{1}".format(prefix, i) #mc = smart.memcached(key) #do an expiry every 10 times if counter_10 >= 7: if all_set == True: mc.delete(key) mc.set(key, 0, 0, payload) if counter_10 == 10: counter_10 = 0 else:
rest = RestConnection(server) nodes = rest.node_statuses() vm = VBucketAwareMemcached(rest,{"name":"bucket-0","password":""}) key = str(uuid.uuid4()) vm.memcached(key).set(key, 0, 0, "hi") vm.memcached(key).get(key) RebalanceHelper.print_taps_from_all_nodes(rest,bucket="bucket-0",password="") RebalanceHelper.verify_items_count(server,"bucket-0") RebalanceHelper.verify_items_count(server,"bucket-1") RebalanceHelper.verify_items_count(server,"bucket-2") RebalanceHelper.wait_till_total_numbers_match(server,"bucket-0",120,"") cm = MemcachedClientHelper.proxy_client(server, "bucket-0", "") key = str(uuid.uuid4()) cm.set(key, 0, 0, "hi") cm.get(key) cm1 = MemcachedClientHelper.direct_client(server, "default", "") key = str(uuid.uuid4()) cm1.set(key, 0, 0, "hi") cm1.get(key) c1 = MemcachedClientHelper.proxy_client(server, "hello", "") key = str(uuid.uuid4()) c1.set(key, 0, 0, "hi")
expiry = 0 if "count" in params: count = int(params["count"]) print(count) if "size" in params: size = int(params["size"]) if "prefix" in params: prefix = params["prefix"] #if "expiry" in params: # expiry = int(params["expiry"]) payload = MemcachedClientHelper.create_value('*', size) rest = RestConnection(server) buckets = rest.get_buckets() for bucket in buckets: smart = VBucketAwareMemcached(rest, bucket.name) mc = MemcachedClientHelper.proxy_client(server, bucket.name) i = 0 counter_10 = 0 all_set = False while i < count: try: key = "{0}-{1}".format(prefix, i) #mc = smart.memcached(key) #do an expiry every 10 times if counter_10 >= 7: if all_set == True: mc.delete(key) mc.set(key, 0, 0, payload) if counter_10 == 10: counter_10 = 0 else:
server.port = 9000 rest = RestConnection(server) nodes = rest.node_statuses() vm = VBucketAwareMemcached(rest, {"name": "bucket-0", "password": ""}) key = str(uuid.uuid4()) vm.memcached(key).set(key, 0, 0, "hi") vm.memcached(key).get(key) RebalanceHelper.print_taps_from_all_nodes(rest, bucket="bucket-0", password="") RebalanceHelper.verify_items_count(server, "bucket-0") RebalanceHelper.verify_items_count(server, "bucket-1") RebalanceHelper.verify_items_count(server, "bucket-2") RebalanceHelper.wait_till_total_numbers_match(server, "bucket-0", 120, "") cm = MemcachedClientHelper.proxy_client(server, "bucket-0", "") key = str(uuid.uuid4()) cm.set(key, 0, 0, "hi") cm.get(key) cm1 = MemcachedClientHelper.direct_client(server, "default", "") key = str(uuid.uuid4()) cm1.set(key, 0, 0, "hi") cm1.get(key) c1 = MemcachedClientHelper.proxy_client(server, "hello", "") key = str(uuid.uuid4()) c1.set(key, 0, 0, "hi") c1.get(key) c2 = MemcachedClientHelper.direct_client(server, "hello", "")