def _send(self):
self.obs_keys.clear() # {server: [keys]}
observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN)
with self._observables.mutex:
for obs in observables:
vbucketid = VbucketHelper.get_vbucket_id(obs.key, self.cfg.get("vbuckets", 0))
obs_key = ObserveRequestKey(obs.key, vbucketid)
server = self._get_server_str(vbucketid)
vals = self.obs_keys.get(server, [])
vals.append(obs_key)
self.obs_keys[server] = vals
reqs = []
for server, keys in self.obs_keys.iteritems():
req = ObserveRequest(keys)
pkt = req.pack()
try:
self.conns[server].s.send(pkt)
except Exception as e:
print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e)
return None
reqs.append(req)
print "reqs::"
print reqs
return reqs
def _send(self):
self.obs_keys.clear() # {server: [keys]}
observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN)
with self._observables.mutex:
for obs in observables:
vbucketid = VbucketHelper.get_vbucket_id(
obs.key, self.cfg.get("vbuckets", 0))
obs_key = ObserveRequestKey(obs.key, vbucketid)
if obs.persist_count > 0:
persist_server = self._get_server_str(vbucketid)
vals = self.obs_keys.get(persist_server, [])
vals.append(obs_key)
self.obs_keys[persist_server] = vals
if not obs.persist_servers:
obs.persist_servers.add(persist_server)
self._observables.put(obs.key, obs)
if obs.repl_count > 0:
repl_servers = self._get_server_str(vbucketid, repl=True)
if len(repl_servers) < obs.repl_count:
print "<%s> not enough number of replication servers to observe"\
% self.__class__.__name__
obs.status = ObserveStatus.OBS_ERROR # mark out this key
self._observables.put(obs.key, obs)
continue
if not obs.repl_servers:
obs.repl_servers.update(repl_servers)
self._observables.put(obs.key, obs)
for server in obs.repl_servers:
vals = self.obs_keys.get(server, [])
vals.append(obs_key)
self.obs_keys[server] = vals
reqs = []
for server, keys in self.obs_keys.iteritems():
req = ObserveRequest(keys)
pkt = req.pack()
try:
self.conns[server].s.send(pkt)
except KeyError as e:
print "<%s> failed to send observe pkt : %s" % (
self.__class__.__name__, e)
self._add_conn(server)
return None
except Exception as e:
print "<%s> failed to send observe pkt : %s" % (
self.__class__.__name__, e)
self._refresh_conns()
return None
reqs.append(req)
print "reqs::"
print reqs
return reqs
def _send(self):
self.obs_keys.clear() # {server: [keys]}
observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN)
with self._observables.mutex:
for obs in observables:
vbucketid = VbucketHelper.get_vbucket_id(obs.key, self.cfg.get("vbuckets", 0))
obs_key = ObserveRequestKey(obs.key, vbucketid)
if obs.persist_count > 0:
persist_server = self._get_server_str(vbucketid)
vals = self.obs_keys.get(persist_server, [])
vals.append(obs_key)
self.obs_keys[persist_server] = vals
if not obs.persist_servers:
obs.persist_servers.add(persist_server)
self._observables.put(obs.key, obs)
if obs.repl_count > 0:
repl_servers = self._get_server_str(vbucketid, repl=True)
if len(repl_servers) < obs.repl_count:
print "<%s> not enough number of replication servers to observe"\
% self.__class__.__name__
obs.status = ObserveStatus.OBS_ERROR # mark out this key
self._observables.put(obs.key, obs)
continue
if not obs.repl_servers:
obs.repl_servers.update(repl_servers)
self._observables.put(obs.key, obs)
for server in obs.repl_servers:
vals = self.obs_keys.get(server, [])
vals.append(obs_key)
self.obs_keys[server] = vals
reqs = []
for server, keys in self.obs_keys.iteritems():
req = ObserveRequest(keys)
pkt = req.pack()
try:
self.conns[server].s.send(pkt)
except KeyError as e:
print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e)
self._add_conn(server)
return None
except Exception as e:
print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e)
self._refresh_conns()
return None
reqs.append(req)
print "reqs::"
print reqs
return reqs
def observe_single(self, key, server="", timeout=0):
"""
send an observe command and get the response back
parse the response afterwards
@return (status, cas)
@status -1 : network error
@status -2 : protocol error
@status ObserveKeyState
"""
cas = ""
if not key:
print "<%s> observe_single: invalid key" % self.__class__.__name__
return -1
vbucketid = \
VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0))
if not server:
server = self._get_server_str(vbucketid)
req_key = ObserveRequestKey(key, vbucketid)
req = ObserveRequest([req_key])
pkt = req.pack()
try:
skt = self.conns[server].s
except KeyError:
print "<%s> observe_single: KeyError: %s" \
% (self.__class__.__name__, server)
self._add_conn(server)
return -1, cas
try:
SocketHelper.send_bytes(skt, pkt, timeout)
except IOError:
print "<%s> observe_single: IOError: " \
"failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
self._reconnect(self.conns[server])
self._refresh_conns()
return -1, cas
except socket.timeout:
print "<%s> observe_single: timeout: " \
"failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
return -1, cas
except Exception as e:
print "<%s> observe_single: failed to send observe pkt : %s" \
% (self.__class__.__name__, e)
return -1, cas
try:
hdr = SocketHelper.recv_bytes(
skt, ObservePktFmt.OBS_RES_HDR_LEN, timeout)
res = ObserveResponse()
if not res.unpack_hdr(hdr):
if res.status == ERR_NOT_MY_VBUCKET:
self._refresh_conns()
return -1, cas
body = SocketHelper.recv_bytes(skt, res.body_len, timeout)
res.unpack_body(body)
except IOError:
print "<%s> observe_single: IOError: failed to recv observe pkt" \
% self.__class__.__name__
self._reconnect(self.conns[server])
self._refresh_conns()
return -1, cas
except socket.timeout:
print "<%s> observe_single: timeout: failed to recv observe pkt" \
% self.__class__.__name__
return -1, cas
except Exception as e:
print "<%s> observe_single: failed to recv observe pkt : %s" \
% (self.__class__.__name__, e)
return -1, cas
if not res:
print "<%s> observe_single: empty response" \
% self.__class__.__name__
return -1, cas
key_len = len(res.keys)
if key_len != 1:
# we are not supposed to receive responses for more than one key,
# otherwise, it's a server side protocol error
print "<%s> observe_single: invalid number of keys in response: %d"\
% (self.s.__name__, key_len)
return -2, cas
res_key = res.keys[0]
cas = res_key.cas
if res_key.key != key:
print "<%s> observe_single: invalid key %s in response"\
% self.__class__.__name__
return -2, cas
return res_key.key_state, cas
def observe_single(self, key, timeout=0):
"""
send an observe command and get the response back
"""
if not key:
print "<%s> observe_single: invalid key" % self.__class__.__name__
return None
vbucketid = VbucketHelper.get_vbucket_id(key,
self.cfg.get("vbuckets", 0))
server = self._get_server_str(vbucketid)
req_key = ObserveRequestKey(key, vbucketid)
req = ObserveRequest([req_key])
pkt = req.pack()
try:
skt = self.conns[server].s
except KeyError:
print "<%s> KeyError: %s" % (self.__class__.__name__, server)
self._add_conn(server)
return None
try:
SocketHelper.send_bytes(skt, pkt, timeout)
except IOError:
print "<%s> IOError: failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
self._reconnect(self.conns[server])
self._refresh_conns()
return None
except socket.timeout:
print "<%s> timeout: failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
return None
except Exception as e:
print "<%s> failed to send observe pkt : %s" \
% (self.__class__.__name__, e)
return None
try:
hdr = SocketHelper.recv_bytes(skt, ObservePktFmt.OBS_RES_HDR_LEN,
timeout)
res = ObserveResponse()
if not res.unpack_hdr(hdr):
if res.status == ERR_NOT_MY_VBUCKET:
self._refresh_conns()
return None
body = SocketHelper.recv_bytes(skt, res.body_len, timeout)
res.unpack_body(body)
except IOError:
print "<%s> IOError: failed to recv observe pkt" \
% self.__class__.__name__
self._reconnect(self.conns[server])
self._refresh_conns()
return None
except socket.timeout:
print "<%s> timeout: failed to recv observe pkt" \
% self.__class__.__name__
return None
except Exception as e:
print "<%s> failed to recv observe pkt : %s" \
% (self.__class__.__name__, e)
return None
return res
def observe_single(self, key, timeout=0):
"""
send an observe command and get the response back
"""
if not key:
print "<%s> observe_single: invalid key" % self.__class__.__name__
return None
vbucketid = VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0))
server = self._get_server_str(vbucketid)
req_key = ObserveRequestKey(key, vbucketid)
req = ObserveRequest([req_key])
pkt = req.pack()
try:
skt = self.conns[server].s
except KeyError:
print "<%s> KeyError: %s" % (self.__class__.__name__, server)
self._add_conn(server)
return None
try:
SocketHelper.send_bytes(skt, pkt, timeout)
except IOError:
print "<%s> IOError: failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
self._reconnect(self.conns[server])
self._refresh_conns()
return None
except socket.timeout:
print "<%s> timeout: failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
return None
except Exception as e:
print "<%s> failed to send observe pkt : %s" \
% (self.__class__.__name__, e)
return None
try:
hdr = SocketHelper.recv_bytes(skt, ObservePktFmt.OBS_RES_HDR_LEN, timeout)
res = ObserveResponse()
if not res.unpack_hdr(hdr):
if res.status == ERR_NOT_MY_VBUCKET:
self._refresh_conns()
return None
body = SocketHelper.recv_bytes(skt, res.body_len, timeout)
res.unpack_body(body)
except IOError:
print "<%s> IOError: failed to recv observe pkt" \
% self.__class__.__name__
self._reconnect(self.conns[server])
self._refresh_conns()
return None
except socket.timeout:
print "<%s> timeout: failed to recv observe pkt" \
% self.__class__.__name__
return None
except Exception as e:
print "<%s> failed to recv observe pkt : %s" \
% (self.__class__.__name__, e)
return None
return res
def observe_single(self, key, server="", timeout=0):
"""
send an observe command and get the response back
parse the response afterwards
@return (status, cas)
@status -1 : network error
@status -2 : protocol error
@status ObserveKeyState
"""
cas = ""
if not key:
print "<%s> observe_single: invalid key" % self.__class__.__name__
return -1
vbucketid = \
VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0))
if not server:
server = self._get_server_str(vbucketid)
req_key = ObserveRequestKey(key, vbucketid)
req = ObserveRequest([req_key])
pkt = req.pack()
try:
skt = self.conns[server].s
except KeyError:
print "<%s> observe_single: KeyError: %s" \
% (self.__class__.__name__, server)
self._add_conn(server)
return -1, cas
try:
SocketHelper.send_bytes(skt, pkt, timeout)
except IOError:
print "<%s> observe_single: IOError: " \
"failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
self._reconnect(self.conns[server])
self._refresh_conns()
return -1, cas
except socket.timeout:
print "<%s> observe_single: timeout: " \
"failed to send observe pkt : %s" \
% (self.__class__.__name__, pkt)
return -1, cas
except Exception as e:
print "<%s> observe_single: failed to send observe pkt : %s" \
% (self.__class__.__name__, e)
return -1, cas
try:
hdr = SocketHelper.recv_bytes(
skt, ObservePktFmt.OBS_RES_HDR_LEN, timeout)
res = ObserveResponse()
if not res.unpack_hdr(hdr):
if res.status == ERR_NOT_MY_VBUCKET:
self._refresh_conns()
return -1, cas
body = SocketHelper.recv_bytes(skt, res.body_len, timeout)
res.unpack_body(body)
except IOError:
print "<%s> observe_single: IOError: failed to recv observe pkt" \
% self.__class__.__name__
self._reconnect(self.conns[server])
self._refresh_conns()
return -1, cas
except socket.timeout:
print "<%s> observe_single: timeout: failed to recv observe pkt" \
% self.__class__.__name__
return -1, cas
except Exception as e:
print "<%s> observe_single: failed to recv observe pkt : %s" \
% (self.__class__.__name__, e)
return -1, cas
if not res:
print "<%s> observe_single: empty response" \
% self.__class__.__name__
return -1, cas
key_len = len(res.keys)
if key_len != 1:
# we are not supposed to receive responses for more than one key,
# otherwise, it's a server side protocol error
print "<%s> observe_single: invalid number of keys in response: %d"\
% (self.s.__name__, key_len)
return -2, cas
res_key = res.keys[0]
cas = res_key.cas
if res_key.key != key:
print "<%s> observe_single: invalid key %s in response"\
% self.__class__.__name__
return -2, cas
return res_key.key_state, cas
