def _send(self):
self.obs_keys.clear() # {server: [keys]}
observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN)
with self._observables.mutex:
for obs in observables:
vbucketid = VbucketHelper.get_vbucket_id(obs.key, self.cfg.get("vbuckets", 0))
obs_key = ObserveRequestKey(obs.key, vbucketid)
server = self._get_server_str(vbucketid)
vals = self.obs_keys.get(server, [])
vals.append(obs_key)
self.obs_keys[server] = vals
reqs = []
for server, keys in self.obs_keys.iteritems():
req = ObserveRequest(keys)
pkt = req.pack()
try:
self.conns[server].s.send(pkt)
except Exception as e:
print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e)
return None
reqs.append(req)
print "reqs::"
print reqs
return reqs
def _send(self):
self.obs_keys.clear() # {server: [keys]}
observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN)
with self._observables.mutex:
for obs in observables:
vbucketid = VbucketHelper.get_vbucket_id(
obs.key, self.cfg.get("vbuckets", 0))
obs_key = ObserveRequestKey(obs.key, vbucketid)
if obs.persist_count > 0:
persist_server = self._get_server_str(vbucketid)
vals = self.obs_keys.get(persist_server, [])
vals.append(obs_key)
self.obs_keys[persist_server] = vals
if not obs.persist_servers:
obs.persist_servers.add(persist_server)
self._observables.put(obs.key, obs)
if obs.repl_count > 0:
repl_servers = self._get_server_str(vbucketid, repl=True)
if len(repl_servers) < obs.repl_count:
print "<%s> not enough number of replication servers to observe"\
% self.__class__.__name__
obs.status = ObserveStatus.OBS_ERROR # mark out this key
self._observables.put(obs.key, obs)
continue
if not obs.repl_servers:
obs.repl_servers.update(repl_servers)
self._observables.put(obs.key, obs)
for server in obs.repl_servers:
vals = self.obs_keys.get(server, [])
vals.append(obs_key)
self.obs_keys[server] = vals
reqs = []
for server, keys in self.obs_keys.iteritems():
req = ObserveRequest(keys)
pkt = req.pack()
try:
self.conns[server].s.send(pkt)
except KeyError as e:
print "<%s> failed to send observe pkt : %s" % (
self.__class__.__name__, e)
self._add_conn(server)
return None
except Exception as e:
print "<%s> failed to send observe pkt : %s" % (
self.__class__.__name__, e)
self._refresh_conns()
return None
reqs.append(req)
print "reqs::"
print reqs
return reqs
def _send(self):
self.obs_keys.clear() # {server: [keys]}
observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN)
with self._observables.mutex:
for obs in observables:
vbucketid = VbucketHelper.get_vbucket_id(obs.key, self.cfg.get("vbuckets", 0))
obs_key = ObserveRequestKey(obs.key, vbucketid)
if obs.persist_count > 0:
persist_server = self._get_server_str(vbucketid)
vals = self.obs_keys.get(persist_server, [])
vals.append(obs_key)
self.obs_keys[persist_server] = vals
if not obs.persist_servers:
obs.persist_servers.add(persist_server)
self._observables.put(obs.key, obs)
if obs.repl_count > 0:
repl_servers = self._get_server_str(vbucketid, repl=True)
if len(repl_servers) < obs.repl_count:
print "<%s> not enough number of replication servers to observe"\
% self.__class__.__name__
obs.status = ObserveStatus.OBS_ERROR # mark out this key
self._observables.put(obs.key, obs)
continue
if not obs.repl_servers:
obs.repl_servers.update(repl_servers)
self._observables.put(obs.key, obs)
for server in obs.repl_servers:
vals = self.obs_keys.get(server, [])
vals.append(obs_key)
self.obs_keys[server] = vals
reqs = []
for server, keys in self.obs_keys.iteritems():
req = ObserveRequest(keys)
pkt = req.pack()
try:
self.conns[server].s.send(pkt)
except KeyError as e:
print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e)
self._add_conn(server)
return None
except Exception as e:
print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e)
self._refresh_conns()
return None
reqs.append(req)
print "reqs::"
print reqs
return reqs
def observe_single(self, key, server="", timeout=0):
"""
send an observe command and get the response back
parse the response afterwards
@return (status, cas)
@status -1 : network error
@status -2 : protocol error
@status ObserveKeyState
"""
cas = ""
if not key:
print "<%s> observe_single: invalid key" % self.__class__.__name__
return -1
vbucketid = \
VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0))
if not server:
server = self._get_server_str(vbucketid)
req_key = ObserveRequestKey(key, vbucketid)
req = ObserveRequest([req_key])
pkt = req.pack()
try:
skt = self.conns[server].s
except KeyError:
print "<%s> observe_single: KeyError: %s" \
% (self.__class__.__name__, server)
self._add_conn(server)
return -1, cas
try:
SocketHelper.send_bytes(skt, pkt, timeout)
except IOError:
print "<%s> observe_single: IOError: " \
"failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
self._reconnect(self.conns[server])
self._refresh_conns()
return -1, cas
except socket.timeout:
print "<%s> observe_single: timeout: " \
"failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
return -1, cas
except Exception as e:
print "<%s> observe_single: failed to send observe pkt : %s" \
% (self.__class__.__name__, e)
return -1, cas
try:
hdr = SocketHelper.recv_bytes(
skt, ObservePktFmt.OBS_RES_HDR_LEN, timeout)
res = ObserveResponse()
if not res.unpack_hdr(hdr):
if res.status == ERR_NOT_MY_VBUCKET:
self._refresh_conns()
return -1, cas
body = SocketHelper.recv_bytes(skt, res.body_len, timeout)
res.unpack_body(body)
except IOError:
print "<%s> observe_single: IOError: failed to recv observe pkt" \
% self.__class__.__name__
self._reconnect(self.conns[server])
self._refresh_conns()
return -1, cas
except socket.timeout:
print "<%s> observe_single: timeout: failed to recv observe pkt" \
% self.__class__.__name__
return -1, cas
except Exception as e:
print "<%s> observe_single: failed to recv observe pkt : %s" \
% (self.__class__.__name__, e)
return -1, cas
if not res:
print "<%s> observe_single: empty response" \
% self.__class__.__name__
return -1, cas
key_len = len(res.keys)
if key_len != 1:
# we are not supposed to receive responses for more than one key,
# otherwise, it's a server side protocol error
print "<%s> observe_single: invalid number of keys in response: %d"\
% (self.s.__name__, key_len)
return -2, cas
res_key = res.keys[0]
cas = res_key.cas
if res_key.key != key:
print "<%s> observe_single: invalid key %s in response"\
% self.__class__.__name__
return -2, cas
return res_key.key_state, cas
def block_for_replication(self, key, cas, num=1, timeout=0, persist=False):
"""
observe a key until it has been replicated to @param num of servers
@param persist : block until item has been persisted to disk
"""
if not isinstance(num, int) or num <= 0:
print "<%s> block_for_replication: invalid num %s" \
% (self.__class__.__name__, num)
return False
vbucketid = \
VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0))
repl_servers = self._get_server_str(vbucketid, repl=True)
if persist and not self.block_for_persistence(key, cas):
return False
self.backoff = self.cfg.get('obs-backoff', BACKOFF)
print "<%s> block_for_replication: repl_servers: %s,"\
" key: %s, cas: %s, vbucketid: %s" \
% (self.__class__.__name__, repl_servers, key, cas, vbucketid)
while len(repl_servers) >= num > 0:
for server in repl_servers:
if num == 0:
break
status, new_cas = self.observe_single(key, server, timeout)
if status < 0:
repl_servers.remove(server)
continue
if new_cas and new_cas != cas:
# Due to the current protocol limitations,
# assume key is unique and new, skip this server
repl_servers.remove(server)
continue
if status == ObserveKeyState.OBS_PERSISITED:
num -= 1
repl_servers.remove(server)
continue
elif status == ObserveKeyState.OBS_FOUND:
if not persist:
num -= 1
repl_servers.remove(server)
continue
elif status == ObserveKeyState.OBS_NOT_FOUND:
pass
if len(repl_servers) == 1:
sleep(self.backoff)
self.backoff = min(self.backoff * 2, self.max_backoff)
if num > 0:
return False
return True
def observe_single(self, key, timeout=0):
"""
send an observe command and get the response back
"""
if not key:
print "<%s> observe_single: invalid key" % self.__class__.__name__
return None
vbucketid = VbucketHelper.get_vbucket_id(key,
self.cfg.get("vbuckets", 0))
server = self._get_server_str(vbucketid)
req_key = ObserveRequestKey(key, vbucketid)
req = ObserveRequest([req_key])
pkt = req.pack()
try:
skt = self.conns[server].s
except KeyError:
print "<%s> KeyError: %s" % (self.__class__.__name__, server)
self._add_conn(server)
return None
try:
SocketHelper.send_bytes(skt, pkt, timeout)
except IOError:
print "<%s> IOError: failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
self._reconnect(self.conns[server])
self._refresh_conns()
return None
except socket.timeout:
print "<%s> timeout: failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
return None
except Exception as e:
print "<%s> failed to send observe pkt : %s" \
% (self.__class__.__name__, e)
return None
try:
hdr = SocketHelper.recv_bytes(skt, ObservePktFmt.OBS_RES_HDR_LEN,
timeout)
res = ObserveResponse()
if not res.unpack_hdr(hdr):
if res.status == ERR_NOT_MY_VBUCKET:
self._refresh_conns()
return None
body = SocketHelper.recv_bytes(skt, res.body_len, timeout)
res.unpack_body(body)
except IOError:
print "<%s> IOError: failed to recv observe pkt" \
% self.__class__.__name__
self._reconnect(self.conns[server])
self._refresh_conns()
return None
except socket.timeout:
print "<%s> timeout: failed to recv observe pkt" \
% self.__class__.__name__
return None
except Exception as e:
print "<%s> failed to recv observe pkt : %s" \
% (self.__class__.__name__, e)
return None
return res
def observe_single(self, key, timeout=0):
"""
send an observe command and get the response back
"""
if not key:
print "<%s> observe_single: invalid key" % self.__class__.__name__
return None
vbucketid = VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0))
server = self._get_server_str(vbucketid)
req_key = ObserveRequestKey(key, vbucketid)
req = ObserveRequest([req_key])
pkt = req.pack()
try:
skt = self.conns[server].s
except KeyError:
print "<%s> KeyError: %s" % (self.__class__.__name__, server)
self._add_conn(server)
return None
try:
SocketHelper.send_bytes(skt, pkt, timeout)
except IOError:
print "<%s> IOError: failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
self._reconnect(self.conns[server])
self._refresh_conns()
return None
except socket.timeout:
print "<%s> timeout: failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
return None
except Exception as e:
print "<%s> failed to send observe pkt : %s" \
% (self.__class__.__name__, e)
return None
try:
hdr = SocketHelper.recv_bytes(skt, ObservePktFmt.OBS_RES_HDR_LEN, timeout)
res = ObserveResponse()
if not res.unpack_hdr(hdr):
if res.status == ERR_NOT_MY_VBUCKET:
self._refresh_conns()
return None
body = SocketHelper.recv_bytes(skt, res.body_len, timeout)
res.unpack_body(body)
except IOError:
print "<%s> IOError: failed to recv observe pkt" \
% self.__class__.__name__
self._reconnect(self.conns[server])
self._refresh_conns()
return None
except socket.timeout:
print "<%s> timeout: failed to recv observe pkt" \
% self.__class__.__name__
return None
except Exception as e:
print "<%s> failed to recv observe pkt : %s" \
% (self.__class__.__name__, e)
return None
return res
def observe_single(self, key, server="", timeout=0):
"""
send an observe command and get the response back
parse the response afterwards
@return (status, cas)
@status -1 : network error
@status -2 : protocol error
@status ObserveKeyState
"""
cas = ""
if not key:
print "<%s> observe_single: invalid key" % self.__class__.__name__
return -1
vbucketid = \
VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0))
if not server:
server = self._get_server_str(vbucketid)
req_key = ObserveRequestKey(key, vbucketid)
req = ObserveRequest([req_key])
pkt = req.pack()
try:
skt = self.conns[server].s
except KeyError:
print "<%s> observe_single: KeyError: %s" \
% (self.__class__.__name__, server)
self._add_conn(server)
return -1, cas
try:
SocketHelper.send_bytes(skt, pkt, timeout)
except IOError:
print "<%s> observe_single: IOError: " \
"failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
self._reconnect(self.conns[server])
self._refresh_conns()
return -1, cas
except socket.timeout:
print "<%s> observe_single: timeout: " \
"failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
return -1, cas
except Exception as e:
print "<%s> observe_single: failed to send observe pkt : %s" \
% (self.__class__.__name__, e)
return -1, cas
try:
hdr = SocketHelper.recv_bytes(
skt, ObservePktFmt.OBS_RES_HDR_LEN, timeout)
res = ObserveResponse()
if not res.unpack_hdr(hdr):
if res.status == ERR_NOT_MY_VBUCKET:
self._refresh_conns()
return -1, cas
body = SocketHelper.recv_bytes(skt, res.body_len, timeout)
res.unpack_body(body)
except IOError:
print "<%s> observe_single: IOError: failed to recv observe pkt" \
% self.__class__.__name__
self._reconnect(self.conns[server])
self._refresh_conns()
return -1, cas
except socket.timeout:
print "<%s> observe_single: timeout: failed to recv observe pkt" \
% self.__class__.__name__
return -1, cas
except Exception as e:
print "<%s> observe_single: failed to recv observe pkt : %s" \
% (self.__class__.__name__, e)
return -1, cas
if not res:
print "<%s> observe_single: empty response" \
% self.__class__.__name__
return -1, cas
key_len = len(res.keys)
if key_len != 1:
# we are not supposed to receive responses for more than one key,
# otherwise, it's a server side protocol error
print "<%s> observe_single: invalid number of keys in response: %d"\
% (self.s.__name__, key_len)
return -2, cas
res_key = res.keys[0]
cas = res_key.cas
if res_key.key != key:
print "<%s> observe_single: invalid key %s in response"\
% self.__class__.__name__
return -2, cas
return res_key.key_state, cas
def block_for_replication(self, key, cas, num=1, timeout=0, persist=False):
"""
observe a key until it has been replicated to @param num of servers
@param persist : block until item has been persisted to disk
"""
if not isinstance(num, int) or num <= 0:
print "<%s> block_for_replication: invalid num %s" \
% (self.__class__.__name__, num)
return False
vbucketid = \
VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0))
repl_servers = self._get_server_str(vbucketid, repl=True)
if persist and not self.block_for_persistence(key, cas):
return False
self.backoff = self.cfg.get('obs-backoff', BACKOFF)
print "<%s> block_for_replication: repl_servers: %s,"\
" key: %s, cas: %s, vbucketid: %s" \
% (self.__class__.__name__, repl_servers, key, cas, vbucketid)
while len(repl_servers) >= num > 0:
for server in repl_servers:
if num == 0:
break
status, new_cas = self.observe_single(key, server, timeout)
if status < 0:
repl_servers.remove(server)
continue
if new_cas and new_cas != cas:
# Due to the current protocol limitations,
# assume key is unique and new, skip this server
repl_servers.remove(server)
continue
if status == ObserveKeyState.OBS_PERSISITED:
num -= 1
repl_servers.remove(server)
continue
elif status == ObserveKeyState.OBS_FOUND:
if not persist:
num -= 1
repl_servers.remove(server)
continue
elif status == ObserveKeyState.OBS_NOT_FOUND:
pass
if len(repl_servers) == 1:
sleep(self.backoff)
self.backoff = min(self.backoff * 2, self.max_backoff)
if num > 0:
return False
return True
