def _getr_items(self, item_count, replica_count, prefix, vprefix=""):
time_start = time.time()
get_count = 0
last_error = ""
error_count = 0
awareness = VBucketAwareMemcached(self.rest, self.default_bucket_name)
for r in range(replica_count):
for i in range(item_count):
retry = True
key = prefix + "_key_" + str(i)
while retry:
client = awareness.memcached(key, r)
try:
value = client.getr(prefix + "_key_" + str(i))[2]
assert(value == vprefix + "_value_" + str(i))
get_count += 1
retry = False
except mc_bin_client.MemcachedError as e:
last_error = "failed to getr key {0}, error: {1}".format(prefix + "_key_" + str(i), e)
error_count += 1
if e.status == 7:
self.log.info("getting new vbucket map {0}")
awareness.reset(self.rest)
else:
retry = False
except Exception as e:
last_error = "failed to getr key {0}, error: {1}".format(prefix + "_key_" + str(i), e)
error_count += 1
retry = False
if error_count > 0:
self.log.error("got {0} errors, last error: {1}".format(error_count, last_error))
self.log.info("got {0} replica items in {1} seconds".format(get_count, time.time() - time_start))
awareness.done()
return get_count
class StoreMembaseBinary(StoreMemcachedBinary):
def connect_host_port(self, host, port, user, pswd):
from membase.api.rest_client import RestConnection
from memcached.helper.data_helper import VBucketAwareMemcached
info = { "ip": host, "port": port,
'username': user or 'Administrator',
'password': pswd or 'password' }
rest = RestConnection(info)
self.awareness = VBucketAwareMemcached(rest, user or 'default', info)
self.backoff = 0
self.xfer_sent = 0
self.xfer_recv = 0
def flush_level(self):
f = StoreMemcachedBinary.flush_level(self)
return f * len(self.awareness.memcacheds)
def inflight_start(self):
return { 's_bufs': {}, # Key is server str, value is [] of buffer.
's_cmds': {} # Key is server str, value is int (number of cmds).
}
def inflight_complete(self, inflight_grp):
rv = [] # Array of tuples (server, buffer).
s_bufs = inflight_grp['s_bufs']
for server in s_bufs.keys():
buffers = s_bufs[server]
rv.append((server, ''.join(buffers)))
return rv
def inflight_send(self, inflight_msg):
sent = 0
for server, buf in inflight_msg:
try:
conn = self.awareness.memcacheds[server]
conn.s.send(buf)
sent += len(buf)
except:
pass
return sent
def inflight_recv(self, inflight, inflight_grp, expectBuffer=None):
received = 0
s_cmds = inflight_grp['s_cmds']
reset_my_awareness = False
backoff = False
for server in s_cmds.keys():
try:
conn = self.awareness.memcacheds[server]
try:
recvBuf = conn.recvBuf
except:
recvBuf = ''
if expectBuffer == False and recvBuf != '':
raise Exception("Was expecting empty buffer, but have (" + \
str(len(recvBuf)) + "): " + recvBuf)
cmds = s_cmds[server]
for i in range(cmds):
try:
rcmd, keylen, extralen, errcode, datalen, ropaque, val, recvBuf = \
self.recvMsgSockBuf(conn.s, recvBuf)
received += datalen + MIN_RECV_PACKET
if errcode == ERR_NOT_MY_VBUCKET:
reset_my_awareness = True
elif errcode == ERR_ENOMEM or \
errcode == ERR_EBUSY or \
errcode == ERR_ETMPFAIL:
backoff = True
except:
reset_my_awareness = True
backoff = True
conn.recvBuf = recvBuf
except:
reset_my_awareness = True
backoff = True
if backoff:
self.backoff = max(self.backoff, 0.1) * \
self.cfg.get('backoff-factor', 2.0)
if self.backoff > 0:
self.cur['cur-backoffs'] = self.cur.get('cur-backoffs', 0) + 1
time.sleep(self.backoff)
else:
self.backoff = 0
if reset_my_awareness:
try:
self.awareness.reset()
except:
pass
return received
def recvMsgSockBuf(self, sock, buf):
pkt, buf = self.readbytes(sock, MIN_RECV_PACKET, buf)
magic, cmd, keylen, extralen, dtype, errcode, datalen, opaque, cas = \
struct.unpack(RES_PKT_FMT, pkt)
if magic != RES_MAGIC_BYTE:
raise Exception("Unexpected recvMsg magic: " + str(magic))
val, buf = self.readbytes(sock, datalen, buf)
return cmd, keylen, extralen, errcode, datalen, opaque, val, buf
def inflight_append_buffer(self, grp, vbucketId, opcode, opaque):
s_bufs = grp['s_bufs']
s_cmds = grp['s_cmds']
s = self.awareness.vBucketMap[vbucketId]
m = s_bufs.get(s, None)
if m is None:
m = []
s_bufs[s] = m
s_cmds[s] = 0
s_cmds[s] += 1
return m
class StoreMembaseBinary(StoreMemcachedBinary):
def connect_host_port(self, host, port, user, pswd, bucket="default"):
"""
Connect to the server host using REST API.
Username and password should be rest_username and rest_password, \
generally they are different from ssh identities.
"""
from membase.api.rest_client import RestConnection
from memcached.helper.data_helper import VBucketAwareMemcached
info = { "ip": host, "port": port,
'username': user or self.cfg.get("rest_username", "Administrator"),
'password': pswd or self.cfg.get("rest_password", "password") }
rest = RestConnection(info)
self.awareness = VBucketAwareMemcached(rest, bucket, info)
self.backoff = 0
self.xfer_sent = 0
self.xfer_recv = 0
def flush_level(self):
f = StoreMemcachedBinary.flush_level(self)
return f * len(self.awareness.memcacheds)
def inflight_start(self):
return { 's_bufs': {}, # Key is server str, value is [] of buffer.
's_cmds': {} # Key is server str, value is int (number of cmds).
}
def inflight_complete(self, inflight_grp):
rv = [] # Array of tuples (server, buffer).
s_bufs = inflight_grp['s_bufs']
for server in s_bufs.keys():
buffers = s_bufs[server]
rv.append((server, ''.join(buffers)))
return rv
def inflight_send(self, inflight_msg):
"""
If timeout value is 0,
blocks until everything been sent out \
or the connection breaks.
"""
timeout_sec = self.cfg.get("socket-timeout", 0)
sent_total = 0
for server, buf in inflight_msg:
length = len(buf)
if length == 0:
continue
sent_tuple = 0 # byte sent out per tuple in inflight_msg
while sent_tuple < length:
try:
conn = self.awareness.memcacheds[server]
if timeout_sec > 0:
conn.s.settimeout(timeout_sec)
sent = conn.s.send(buf)
if sent == 0:
log.error("[mcsoda] StoreMembaseBinary.send-zero / skt.send returned 0.")
self.cur["cur-StoreMembaseBinary.send-zero"] = self.cur.get("cur-StoreMembaseBinary.send-zero", 0) + 1
break
sent_tuple += sent
except socket.timeout:
log.error("[mcsoda] EXCEPTION: StoreMembaseBinary.send-socket.timeout / inflight_send timed out")
self.cur["cur-ex-StoreMembaseBinary.send-socket.timeout"] = self.cur.get("cur-ex-StoreMembaseBinary.send-socket.timeout", 0) + 1
break
except Exception as e:
log.error("[mcsoda] EXCEPTION: StoreMembaseBinary.send / inflight_send: " + str(e))
self.cur["cur-ex-StoreMembaseBinary.send"] = self.cur.get("cur-ex-StoreMembaseBinary.send", 0) + 1
break
sent_total += sent_tuple
return sent_total
def inflight_recv(self, inflight, inflight_grp, expectBuffer=None):
received = 0
s_cmds = inflight_grp['s_cmds']
reset_my_awareness = False
backoff = False
for server in s_cmds.keys():
try:
conn = self.awareness.memcacheds[server]
try:
recvBuf = conn.recvBuf
except:
recvBuf = ''
if expectBuffer == False and recvBuf != '':
raise Exception("Was expecting empty buffer, but have (" + \
str(len(recvBuf)) + "): " + recvBuf)
cmds = s_cmds[server]
for i in range(cmds):
try:
rcmd, keylen, extralen, errcode, datalen, ropaque, val, recvBuf = \
self.recvMsgSockBuf(conn.s, recvBuf)
received += datalen + MIN_RECV_PACKET
if errcode == ERR_NOT_MY_VBUCKET:
reset_my_awareness = True
elif errcode == ERR_ENOMEM or \
errcode == ERR_EBUSY or \
errcode == ERR_ETMPFAIL:
backoff = True
log.error("[mcsoda] inflight recv errorcode = %s" %errcode)
except Exception as e:
log.error("[mcsoda] EXCEPTION: StoreMembaseBinary.recvMsgSockBuf / inflight_recv inner: " + str(e))
self.cur["cur-ex-StoreMembaseBinary.recvMsgSockBuf"] = self.cur.get("cur-ex-StoreMembaseBinary.recvMsgSockBuf", 0) + 1
reset_my_awareness = True
backoff = True
conn.recvBuf = recvBuf
except Exception as e:
log.error("[mcsoda] EXCEPTION: StoreMembaseBinary.inflight_recv / outer: " + str(e))
self.cur["cur-ex-StoreMembaseBinary.inflight_recv"] = self.cur.get("cur-ex-StoreMembaseBinary.inflight_recv", 0) + 1
reset_my_awareness = True
backoff = True
if backoff:
self.backoff = max(self.backoff, 0.1) * \
self.cfg.get('backoff-factor', 2.0)
if self.backoff > 0:
self.cur['cur-backoffs'] = self.cur.get('cur-backoffs', 0) + 1
log.info("[mcsoda] inflight recv backoff = %s" %self.backoff)
time.sleep(self.backoff)
else:
self.backoff = 0
if reset_my_awareness:
try:
self.awareness.reset()
except Exception as e:
log.error("[mcsoda] EXCEPTION: StoreMembaseBinary.awareness.reset: " + str(e))
self.cur["cur-ex-StoreMembaseBinary.awareness.reset"] = self.cur.get("cur-ex-StoreMembaseBinary.awareness.reset", 0) + 1
print "EXCEPTION: self.awareness.reset()"
pass
return received
def recvMsgSockBuf(self, sock, buf):
pkt, buf = self.readbytes(sock, MIN_RECV_PACKET, buf)
magic, cmd, keylen, extralen, dtype, errcode, datalen, opaque, cas = \
struct.unpack(RES_PKT_FMT, pkt)
if magic != RES_MAGIC_BYTE:
raise Exception("Unexpected recvMsg magic: " + str(magic))
val, buf = self.readbytes(sock, datalen, buf)
return cmd, keylen, extralen, errcode, datalen, opaque, val, buf
def inflight_append_buffer(self, grp, vbucketId, opcode, opaque):
s_bufs = grp['s_bufs']
s_cmds = grp['s_cmds']
s = self.awareness.vBucketMap[vbucketId]
m = s_bufs.get(s, None)
if m is None:
m = []
s_bufs[s] = m
s_cmds[s] = 0
s_cmds[s] += 1
return m
class StoreMembaseBinary(StoreMemcachedBinary):
def connect_host_port(self, host, port, user, pswd):
from membase.api.rest_client import RestConnection
from memcached.helper.data_helper import VBucketAwareMemcached
info = { "ip": host, "port": port,
'username': user or 'Administrator',
'password': pswd or 'password' }
rest = RestConnection(info)
self.awareness = VBucketAwareMemcached(rest, user or 'default', info)
self.backoff = 0
self.xfer_sent = 0
self.xfer_recv = 0
def flush_level(self):
f = StoreMemcachedBinary.flush_level(self)
return f * len(self.awareness.memcacheds)
def inflight_start(self):
return { 's_bufs': {}, # Key is server str, value is [] of buffer.
's_cmds': {} # Key is server str, value is int (number of cmds).
}
def inflight_complete(self, inflight_grp):
rv = [] # Array of tuples (server, buffer).
s_bufs = inflight_grp['s_bufs']
for server in s_bufs.keys():
buffers = s_bufs[server]
rv.append((server, ''.join(buffers)))
return rv
def inflight_send(self, inflight_msg):
sent = 0
for server, buf in inflight_msg:
try:
conn = self.awareness.memcacheds[server]
conn.s.send(buf)
sent += len(buf)
except:
pass
return sent
def inflight_recv(self, inflight, inflight_grp, expectBuffer=None):
received = 0
s_cmds = inflight_grp['s_cmds']
reset_my_awareness = False
backoff = False
for server in s_cmds.keys():
try:
conn = self.awareness.memcacheds[server]
try:
recvBuf = conn.recvBuf
except:
recvBuf = ''
if expectBuffer == False and recvBuf != '':
raise Exception("Was expecting empty buffer, but have (" + \
str(len(recvBuf)) + "): " + recvBuf)
cmds = s_cmds[server]
for i in range(cmds):
try:
rcmd, keylen, extralen, errcode, datalen, ropaque, val, recvBuf = \
self.recvMsgSockBuf(conn.s, recvBuf)
received += datalen + MIN_RECV_PACKET
if errcode == ERR_NOT_MY_VBUCKET:
reset_my_awareness = True
elif errcode == ERR_ENOMEM or \
errcode == ERR_EBUSY or \
errcode == ERR_ETMPFAIL:
backoff = True
except:
reset_my_awareness = True
backoff = True
conn.recvBuf = recvBuf
except:
reset_my_awareness = True
backoff = True
if backoff:
self.backoff = max(self.backoff, 0.1) * \
self.cfg.get('backoff-factor', 2.0)
if self.backoff > 0:
self.cur['cur-backoffs'] = self.cur.get('cur-backoffs', 0) + 1
time.sleep(self.backoff)
else:
self.backoff = 0
if reset_my_awareness:
try:
self.awareness.reset()
except:
pass
return received
def recvMsgSockBuf(self, sock, buf):
pkt, buf = self.readbytes(sock, MIN_RECV_PACKET, buf)
magic, cmd, keylen, extralen, dtype, errcode, datalen, opaque, cas = \
struct.unpack(RES_PKT_FMT, pkt)
if magic != RES_MAGIC_BYTE:
raise Exception("Unexpected recvMsg magic: " + str(magic))
val, buf = self.readbytes(sock, datalen, buf)
return cmd, keylen, extralen, errcode, datalen, opaque, val, buf
def inflight_append_buffer(self, grp, vbucketId, opcode, opaque):
s_bufs = grp['s_bufs']
s_cmds = grp['s_cmds']
s = self.awareness.vBucketMap[vbucketId]
m = s_bufs.get(s, None)
if m is None:
m = []
s_bufs[s] = m
s_cmds[s] = 0
s_cmds[s] += 1
return m