def _getr_items(self, item_count, replica_count, prefix, vprefix=""): time_start = time.time() get_count = 0 last_error = "" error_count = 0 awareness = VBucketAwareMemcached(self.rest, self.default_bucket_name) for r in range(replica_count): for i in range(item_count): retry = True key = prefix + "_key_" + str(i) while retry: client = awareness.memcached(key, r) try: value = client.getr(prefix + "_key_" + str(i))[2] assert(value == vprefix + "_value_" + str(i)) get_count += 1 retry = False except mc_bin_client.MemcachedError as e: last_error = "failed to getr key {0}, error: {1}".format(prefix + "_key_" + str(i), e) error_count += 1 if e.status == 7: self.log.info("getting new vbucket map {0}") awareness.reset(self.rest) else: retry = False except Exception as e: last_error = "failed to getr key {0}, error: {1}".format(prefix + "_key_" + str(i), e) error_count += 1 retry = False if error_count > 0: self.log.error("got {0} errors, last error: {1}".format(error_count, last_error)) self.log.info("got {0} replica items in {1} seconds".format(get_count, time.time() - time_start)) awareness.done() return get_count
class StoreMembaseBinary(StoreMemcachedBinary): def connect_host_port(self, host, port, user, pswd): from membase.api.rest_client import RestConnection from memcached.helper.data_helper import VBucketAwareMemcached info = { "ip": host, "port": port, 'username': user or 'Administrator', 'password': pswd or 'password' } rest = RestConnection(info) self.awareness = VBucketAwareMemcached(rest, user or 'default', info) self.backoff = 0 self.xfer_sent = 0 self.xfer_recv = 0 def flush_level(self): f = StoreMemcachedBinary.flush_level(self) return f * len(self.awareness.memcacheds) def inflight_start(self): return { 's_bufs': {}, # Key is server str, value is [] of buffer. 's_cmds': {} # Key is server str, value is int (number of cmds). } def inflight_complete(self, inflight_grp): rv = [] # Array of tuples (server, buffer). s_bufs = inflight_grp['s_bufs'] for server in s_bufs.keys(): buffers = s_bufs[server] rv.append((server, ''.join(buffers))) return rv def inflight_send(self, inflight_msg): sent = 0 for server, buf in inflight_msg: try: conn = self.awareness.memcacheds[server] conn.s.send(buf) sent += len(buf) except: pass return sent def inflight_recv(self, inflight, inflight_grp, expectBuffer=None): received = 0 s_cmds = inflight_grp['s_cmds'] reset_my_awareness = False backoff = False for server in s_cmds.keys(): try: conn = self.awareness.memcacheds[server] try: recvBuf = conn.recvBuf except: recvBuf = '' if expectBuffer == False and recvBuf != '': raise Exception("Was expecting empty buffer, but have (" + \ str(len(recvBuf)) + "): " + recvBuf) cmds = s_cmds[server] for i in range(cmds): try: rcmd, keylen, extralen, errcode, datalen, ropaque, val, recvBuf = \ self.recvMsgSockBuf(conn.s, recvBuf) received += datalen + MIN_RECV_PACKET if errcode == ERR_NOT_MY_VBUCKET: reset_my_awareness = True elif errcode == ERR_ENOMEM or \ errcode == ERR_EBUSY or \ errcode == ERR_ETMPFAIL: backoff = True except: reset_my_awareness = True backoff = True conn.recvBuf = recvBuf except: reset_my_awareness = True backoff = True if backoff: self.backoff = max(self.backoff, 0.1) * \ self.cfg.get('backoff-factor', 2.0) if self.backoff > 0: self.cur['cur-backoffs'] = self.cur.get('cur-backoffs', 0) + 1 time.sleep(self.backoff) else: self.backoff = 0 if reset_my_awareness: try: self.awareness.reset() except: pass return received def recvMsgSockBuf(self, sock, buf): pkt, buf = self.readbytes(sock, MIN_RECV_PACKET, buf) magic, cmd, keylen, extralen, dtype, errcode, datalen, opaque, cas = \ struct.unpack(RES_PKT_FMT, pkt) if magic != RES_MAGIC_BYTE: raise Exception("Unexpected recvMsg magic: " + str(magic)) val, buf = self.readbytes(sock, datalen, buf) return cmd, keylen, extralen, errcode, datalen, opaque, val, buf def inflight_append_buffer(self, grp, vbucketId, opcode, opaque): s_bufs = grp['s_bufs'] s_cmds = grp['s_cmds'] s = self.awareness.vBucketMap[vbucketId] m = s_bufs.get(s, None) if m is None: m = [] s_bufs[s] = m s_cmds[s] = 0 s_cmds[s] += 1 return m
class StoreMembaseBinary(StoreMemcachedBinary): def connect_host_port(self, host, port, user, pswd, bucket="default"): """ Connect to the server host using REST API. Username and password should be rest_username and rest_password, \ generally they are different from ssh identities. """ from membase.api.rest_client import RestConnection from memcached.helper.data_helper import VBucketAwareMemcached info = { "ip": host, "port": port, 'username': user or self.cfg.get("rest_username", "Administrator"), 'password': pswd or self.cfg.get("rest_password", "password") } rest = RestConnection(info) self.awareness = VBucketAwareMemcached(rest, bucket, info) self.backoff = 0 self.xfer_sent = 0 self.xfer_recv = 0 def flush_level(self): f = StoreMemcachedBinary.flush_level(self) return f * len(self.awareness.memcacheds) def inflight_start(self): return { 's_bufs': {}, # Key is server str, value is [] of buffer. 's_cmds': {} # Key is server str, value is int (number of cmds). } def inflight_complete(self, inflight_grp): rv = [] # Array of tuples (server, buffer). s_bufs = inflight_grp['s_bufs'] for server in s_bufs.keys(): buffers = s_bufs[server] rv.append((server, ''.join(buffers))) return rv def inflight_send(self, inflight_msg): """ If timeout value is 0, blocks until everything been sent out \ or the connection breaks. """ timeout_sec = self.cfg.get("socket-timeout", 0) sent_total = 0 for server, buf in inflight_msg: length = len(buf) if length == 0: continue sent_tuple = 0 # byte sent out per tuple in inflight_msg while sent_tuple < length: try: conn = self.awareness.memcacheds[server] if timeout_sec > 0: conn.s.settimeout(timeout_sec) sent = conn.s.send(buf) if sent == 0: log.error("[mcsoda] StoreMembaseBinary.send-zero / skt.send returned 0.") self.cur["cur-StoreMembaseBinary.send-zero"] = self.cur.get("cur-StoreMembaseBinary.send-zero", 0) + 1 break sent_tuple += sent except socket.timeout: log.error("[mcsoda] EXCEPTION: StoreMembaseBinary.send-socket.timeout / inflight_send timed out") self.cur["cur-ex-StoreMembaseBinary.send-socket.timeout"] = self.cur.get("cur-ex-StoreMembaseBinary.send-socket.timeout", 0) + 1 break except Exception as e: log.error("[mcsoda] EXCEPTION: StoreMembaseBinary.send / inflight_send: " + str(e)) self.cur["cur-ex-StoreMembaseBinary.send"] = self.cur.get("cur-ex-StoreMembaseBinary.send", 0) + 1 break sent_total += sent_tuple return sent_total def inflight_recv(self, inflight, inflight_grp, expectBuffer=None): received = 0 s_cmds = inflight_grp['s_cmds'] reset_my_awareness = False backoff = False for server in s_cmds.keys(): try: conn = self.awareness.memcacheds[server] try: recvBuf = conn.recvBuf except: recvBuf = '' if expectBuffer == False and recvBuf != '': raise Exception("Was expecting empty buffer, but have (" + \ str(len(recvBuf)) + "): " + recvBuf) cmds = s_cmds[server] for i in range(cmds): try: rcmd, keylen, extralen, errcode, datalen, ropaque, val, recvBuf = \ self.recvMsgSockBuf(conn.s, recvBuf) received += datalen + MIN_RECV_PACKET if errcode == ERR_NOT_MY_VBUCKET: reset_my_awareness = True elif errcode == ERR_ENOMEM or \ errcode == ERR_EBUSY or \ errcode == ERR_ETMPFAIL: backoff = True log.error("[mcsoda] inflight recv errorcode = %s" %errcode) except Exception as e: log.error("[mcsoda] EXCEPTION: StoreMembaseBinary.recvMsgSockBuf / inflight_recv inner: " + str(e)) self.cur["cur-ex-StoreMembaseBinary.recvMsgSockBuf"] = self.cur.get("cur-ex-StoreMembaseBinary.recvMsgSockBuf", 0) + 1 reset_my_awareness = True backoff = True conn.recvBuf = recvBuf except Exception as e: log.error("[mcsoda] EXCEPTION: StoreMembaseBinary.inflight_recv / outer: " + str(e)) self.cur["cur-ex-StoreMembaseBinary.inflight_recv"] = self.cur.get("cur-ex-StoreMembaseBinary.inflight_recv", 0) + 1 reset_my_awareness = True backoff = True if backoff: self.backoff = max(self.backoff, 0.1) * \ self.cfg.get('backoff-factor', 2.0) if self.backoff > 0: self.cur['cur-backoffs'] = self.cur.get('cur-backoffs', 0) + 1 log.info("[mcsoda] inflight recv backoff = %s" %self.backoff) time.sleep(self.backoff) else: self.backoff = 0 if reset_my_awareness: try: self.awareness.reset() except Exception as e: log.error("[mcsoda] EXCEPTION: StoreMembaseBinary.awareness.reset: " + str(e)) self.cur["cur-ex-StoreMembaseBinary.awareness.reset"] = self.cur.get("cur-ex-StoreMembaseBinary.awareness.reset", 0) + 1 print "EXCEPTION: self.awareness.reset()" pass return received def recvMsgSockBuf(self, sock, buf): pkt, buf = self.readbytes(sock, MIN_RECV_PACKET, buf) magic, cmd, keylen, extralen, dtype, errcode, datalen, opaque, cas = \ struct.unpack(RES_PKT_FMT, pkt) if magic != RES_MAGIC_BYTE: raise Exception("Unexpected recvMsg magic: " + str(magic)) val, buf = self.readbytes(sock, datalen, buf) return cmd, keylen, extralen, errcode, datalen, opaque, val, buf def inflight_append_buffer(self, grp, vbucketId, opcode, opaque): s_bufs = grp['s_bufs'] s_cmds = grp['s_cmds'] s = self.awareness.vBucketMap[vbucketId] m = s_bufs.get(s, None) if m is None: m = [] s_bufs[s] = m s_cmds[s] = 0 s_cmds[s] += 1 return m