Ejemplo n.º 1
0
    def test_MB_32114(self):
        try:
            from sdk_client import SDKClient
        except:
            from sdk_client3 import SDKClient
        import couchbase.subdocument as SD

        rest = RestConnection(self.master)
        client = VBucketAwareMemcached(rest, 'default')
        if self.maxttl:
            self._expiry_pager(self.master)
        sdk_client = SDKClient(scheme='couchbase',
                               hosts=[self.master.ip],
                               bucket='default')
        KEY_NAME = 'key1'

        for i in range(1000):
            mcd = client.memcached(KEY_NAME + str(i))
            rc = mcd.set(KEY_NAME + str(i), 0, 0,
                         json.dumps({'value': 'value2'}))
            sdk_client.mutate_in(
                KEY_NAME + str(i),
                SD.upsert("subdoc_key",
                          "subdoc_val",
                          xattr=True,
                          create_parents=True))
            # wait for it to persist
            persisted = 0
            while persisted == 0:
                opaque, rep_time, persist_time, persisted, cas = client.observe(
                    KEY_NAME + str(i))

        start_time = time.time()
        self._load_doc_data_all_buckets(batch_size=1000)
        end_time = time.time()

        for i in range(1000):
            try:
                mcd = client.memcached(KEY_NAME + str(i))
                _, flags, exp, seqno, cas = client.memcached(
                    KEY_NAME + str(i)).getMeta(KEY_NAME + str(i))
                rc = mcd.del_with_meta(KEY_NAME + str(i), 0, 0, 2, cas + 1)
            except MemcachedError as exp:
                self.fail("Exception with del_with meta - {0}".format(exp))
        self.cluster.compact_bucket(self.master, "default")
        if self.maxttl:
            time_to_sleep = (self.maxttl - (end_time - start_time)) + 20
            self.sleep(int(time_to_sleep))
        else:
            self.sleep(60)
        active_bucket_items = rest.get_active_key_count("default")
        replica_bucket_items = rest.get_replica_key_count("default")
        print('active_bucket_items ', active_bucket_items)
        print('replica_bucket_items ', replica_bucket_items)
        if active_bucket_items * self.num_replicas != replica_bucket_items:
            self.fail("Mismatch in data !!!")
Ejemplo n.º 2
0
    def do_basic_ops(self):

        KEY_NAME = 'key1'
        KEY_NAME2 = 'key2'
        CAS = 1234
        self.log.info('Starting basic ops')


        rest = RestConnection(self.master)
        client = VBucketAwareMemcached(rest, 'default')
        mcd = client.memcached(KEY_NAME)

        # MB-17231 - incr with full eviction
        rc = mcd.incr(KEY_NAME, 1)
        print 'rc for incr', rc



        # MB-17289 del with meta


        rc = mcd.set(KEY_NAME, 0,0, json.dumps({'value':'value2'}))
        print 'set is', rc
        cas = rc[1]


        # wait for it to persist
        persisted = 0
        while persisted == 0:
                opaque, rep_time, persist_time, persisted, cas = client.observe(KEY_NAME)


        try:
            rc = mcd.evict_key(KEY_NAME)

        except MemcachedError as exp:
            self.fail("Exception with evict meta - {0}".format(exp) )

        CAS = 0xabcd
        # key, value, exp, flags, seqno, remote_cas

        try:
            #key, exp, flags, seqno, cas
            rc = mcd.del_with_meta(KEY_NAME2, 0, 0, 2, CAS)



        except MemcachedError as exp:
            self.fail("Exception with del_with meta - {0}".format(exp) )
Ejemplo n.º 3
0
    def test_MB_36087(self):
        try:
            from sdk_client import SDKClient
        except:
            from sdk_client3 import SDKClient
        import couchbase.subdocument as SD

        g_key = "test_doc"
        bucket_name = "default"
        sdk_client = SDKClient(scheme='couchbase',
                               hosts=[self.master.ip],
                               bucket=bucket_name)
        rest = RestConnection(self.master)
        client = VBucketAwareMemcached(rest, bucket_name)
        for i in range(self.num_items):
            key = g_key + str(i)
            mcd = client.memcached(key)
            rc = mcd.set(key, 0, 0, json.dumps({'value': 'value2'}))
            sdk_client.mutate_in(
                key,
                SD.upsert("subdoc_key",
                          "subdoc_val",
                          xattr=True,
                          create_parents=True))
            # Wait for key to persist
            persisted = 0
            while persisted == 0:
                opaque, rep_time, persist_time, persisted, cas = \
                    client.observe(key)

            time.sleep(10)
            # Evict the key
            try:
                rc = mcd.evict_key(key)
            except MemcachedError as exp:
                self.fail("Exception with evict meta - %s" % exp)

            # Perform del_with_meta
            try:
                mcd = client.memcached(key)
                _, flags, exp, seqno, cas = client.memcached(key).getMeta(key)
                rc = mcd.del_with_meta(key, 0, 0, 2, cas + 1)
            except MemcachedError as exp:
                self.fail("Exception with del_with meta - {0}".format(exp))
Ejemplo n.º 4
0
    def do_basic_ops(self):

        KEY_NAME = 'key1'
        KEY_NAME2 = 'key2'
        CAS = 1234
        self.log.info('Starting basic ops')

        rest = RestConnection(self.master)
        client = VBucketAwareMemcached(rest, 'default')
        mcd = client.memcached(KEY_NAME)

        # MB-17231 - incr with full eviction
        rc = mcd.incr(KEY_NAME, 1)
        print('rc for incr', rc)

        # MB-17289 del with meta

        rc = mcd.set(KEY_NAME, 0, 0, json.dumps({'value': 'value2'}))
        print('set is', rc)
        cas = rc[1]

        # wait for it to persist
        persisted = 0
        while persisted == 0:
            opaque, rep_time, persist_time, persisted, cas = client.observe(
                KEY_NAME)

        time.sleep(10)
        try:
            rc = mcd.evict_key(KEY_NAME)

        except MemcachedError as exp:
            self.fail("Exception with evict meta - {0}".format(exp))

        CAS = 0xabcd
        # key, value, exp, flags, seqno, remote_cas

        try:
            #key, exp, flags, seqno, cas
            rc = mcd.del_with_meta(KEY_NAME2, 0, 0, 2, CAS)

        except MemcachedError as exp:
            self.fail("Exception with del_with meta - {0}".format(exp))
Ejemplo n.º 5
0
    def _run_observe(self):
        tasks = []
        query_set = "true"
        persisted = 0
        mutated = False
        count = 0
        for bucket in self.buckets:
            self.cluster.create_view(self.master, self.default_design_doc,
                                      self.default_view, bucket , self.wait_timeout * 2)
            client = VBucketAwareMemcached(RestConnection(self.master), bucket)
            self.max_time = timedelta(microseconds=0)
            if self.mutate_by == "multi_set":
                key_val = self._create_multi_set_batch()
                client.setMulti(0, 0, key_val)
            keys = ["observe%s" % (i) for i in xrange(self.num_items)]
            for key in keys:
                mutated = False
                while not mutated and count < 60:
                    try:
                        if self.mutate_by == "set":
                            # client.memcached(key).set(key, 0, 0, "set")
                            client.set(key, 0, 0, "setvalue")
                        elif self.mutate_by == "append":
                            client.memcached(key).append(key, "append")
                        elif self.mutate_by == "prepend" :
                            client.memcached(key).prepend(key, "prepend")
                        elif self.mutate_by == "incr":
                            client.memcached(key).incr(key, 1)
                        elif self.mutate_by == "decr":
                            client.memcached(key).decr(key)
                        mutated = True
                        t_start = datetime.now()
                    except MemcachedError as error:
                        if error.status == 134:
                            loaded = False
                            self.log.error("Memcached error 134, wait for 5 seconds and then try again")
                            count += 1
                            time.sleep(5)
                while persisted == 0:
                    opaque, rep_time, persist_time, persisted, cas = client.observe(key)
                t_end = datetime.now()
                self.log.info("##########key:-%s################" % (key))
                self.log.info("Persisted:- %s" % (persisted))
                self.log.info("Persist_Time:- %s" % (rep_time))
                self.log.info("Time2:- %s" % (t_end - t_start))
                if self.max_time <= (t_end - t_start):
                    self.max_time = (t_end - t_start)
                    self.log.info("Max Time taken for observe is :- %s" % self.max_time)
                    self.log.info("Cas Value:- %s" % (cas))
            query = {"stale" : "false", "full_set" : "true", "connection_timeout" : 60000}
            self.cluster.query_view(self.master, "dev_Doc1", self.default_view.name, query, self.num_items, bucket, timeout=self.wait_timeout)
            self.log.info("Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s" % (self.default_view, bucket))
            # check whether observe has to run with delete and delete parallel with observe or not
            if len (self.observe_with) > 0 :
                if self.observe_with == "delete" :
                    self.log.info("Deleting 0- %s number of items" % (self.num_items / 2))
                    self._load_doc_data_all_buckets('delete', 0, self.num_items / 2)
                    query_set = "true"
                elif self.observe_with == "delete_parallel":
                    self.log.info("Deleting Parallel 0- %s number of items" % (self.num_items / 2))
                    tasks = self._async_load_doc_data_all_buckets('delete', 0, self.num_items / 2)
                    query_set = "false"
                for key in keys:
                    opaque, rep_time, persist_time, persisted, cas = client.memcached(key).observe(key)
                    self.log.info("##########key:-%s################" % (key))
                    self.log.info("Persisted:- %s" % (persisted))
                if self.observe_with == "delete_parallel":
                    for task in tasks:
                        task.result()

                query = {"stale" : "false", "full_set" : query_set, "connection_timeout" : 60000}
                self.cluster.query_view(self.master, "dev_Doc1", self.default_view.name, query, self.num_items / 2, bucket, timeout=self.wait_timeout)
                self.log.info("Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s" % (self.default_view, self.default_bucket_name))

        """test_observe_basic_data_load_delete will test observer basic scenario
Ejemplo n.º 6
0
    def _run_observe(self):
        tasks = []
        query_set = "true"
        persisted = 0
        mutated = False
        count = 0
        for bucket in self.buckets:
            self.cluster.create_view(self.master, self.default_design_doc,
                                     self.default_view, bucket,
                                     self.wait_timeout * 2)
            client = VBucketAwareMemcached(RestConnection(self.master), bucket)
            self.max_time = timedelta(microseconds=0)
            if self.mutate_by == "multi_set":
                key_val = self._create_multi_set_batch()
                client.setMulti(0, 0, key_val)
            keys = ["observe%s" % (i) for i in range(self.num_items)]
            for key in keys:
                mutated = False
                while not mutated and count < 60:
                    try:
                        if self.mutate_by == "set":
                            # client.memcached(key).set(key, 0, 0, "set")
                            client.set(key, 0, 0, "setvalue")
                        elif self.mutate_by == "append":
                            client.memcached(key).append(key, "append")
                        elif self.mutate_by == "prepend":
                            client.memcached(key).prepend(key, "prepend")
                        elif self.mutate_by == "incr":
                            client.memcached(key).incr(key, 1)
                        elif self.mutate_by == "decr":
                            client.memcached(key).decr(key)
                        mutated = True
                        t_start = datetime.now()
                    except MemcachedError as error:
                        if error.status == 134:
                            loaded = False
                            self.log.error(
                                "Memcached error 134, wait for 5 seconds and then try again"
                            )
                            count += 1
                            time.sleep(5)
                while persisted == 0:
                    opaque, rep_time, persist_time, persisted, cas = client.observe(
                        key)
                t_end = datetime.now()
                #self.log.info("##########key:-%s################" % (key))
                #self.log.info("Persisted:- %s" % (persisted))
                #self.log.info("Persist_Time:- %s" % (rep_time))
                #self.log.info("Time2:- %s" % (t_end - t_start))
                if self.max_time <= (t_end - t_start):
                    self.max_time = (t_end - t_start)
                    self.log.info("Max Time taken for observe is :- %s" %
                                  self.max_time)
                    self.log.info("Cas Value:- %s" % (cas))
            query = {
                "stale": "false",
                "full_set": "true",
                "connection_timeout": 600000
            }
            self.cluster.query_view(self.master,
                                    "dev_Doc1",
                                    self.default_view.name,
                                    query,
                                    self.num_items,
                                    bucket,
                                    timeout=self.wait_timeout)
            self.log.info(
                "Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s"
                % (self.default_view, bucket))
            # check whether observe has to run with delete and delete parallel with observe or not
            if len(self.observe_with) > 0:
                if self.observe_with == "delete":
                    self.log.info("Deleting 0- %s number of items" %
                                  (self.num_items // 2))
                    self._load_doc_data_all_buckets('delete', 0,
                                                    self.num_items // 2)
                    query_set = "true"
                elif self.observe_with == "delete_parallel":
                    self.log.info("Deleting Parallel 0- %s number of items" %
                                  (self.num_items // 2))
                    tasks = self._async_load_doc_data_all_buckets(
                        'delete', 0, self.num_items // 2)
                    query_set = "false"
                for key in keys:
                    opaque, rep_time, persist_time, persisted, cas = client.memcached(
                        key).observe(key)
                    self.log.info("##########key:-%s################" % (key))
                    self.log.info("Persisted:- %s" % (persisted))
                if self.observe_with == "delete_parallel":
                    for task in tasks:
                        task.result()

                query = {
                    "stale": "false",
                    "full_set": query_set,
                    "connection_timeout": 600000
                }
                self.cluster.query_view(self.master,
                                        "dev_Doc1",
                                        self.default_view.name,
                                        query,
                                        self.num_items // 2,
                                        bucket,
                                        timeout=self.wait_timeout)
                self.log.info(
                    "Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s"
                    % (self.default_view, self.default_bucket_name))
        """test_observe_basic_data_load_delete will test observer basic scenario