def _send(self): self.obs_keys.clear() # {server: [keys]} observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN) with self._observables.mutex: for obs in observables: vbucketid = VbucketHelper.get_vbucket_id(obs.key, self.cfg.get("vbuckets", 0)) obs_key = ObserveRequestKey(obs.key, vbucketid) server = self._get_server_str(vbucketid) vals = self.obs_keys.get(server, []) vals.append(obs_key) self.obs_keys[server] = vals reqs = [] for server, keys in self.obs_keys.iteritems(): req = ObserveRequest(keys) pkt = req.pack() try: self.conns[server].s.send(pkt) except Exception as e: print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e) return None reqs.append(req) print "reqs::" print reqs return reqs
def _send(self): self.obs_keys.clear() # {server: [keys]} observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN) with self._observables.mutex: for obs in observables: vbucketid = VbucketHelper.get_vbucket_id( obs.key, self.cfg.get("vbuckets", 0)) obs_key = ObserveRequestKey(obs.key, vbucketid) if obs.persist_count > 0: persist_server = self._get_server_str(vbucketid) vals = self.obs_keys.get(persist_server, []) vals.append(obs_key) self.obs_keys[persist_server] = vals if not obs.persist_servers: obs.persist_servers.add(persist_server) self._observables.put(obs.key, obs) if obs.repl_count > 0: repl_servers = self._get_server_str(vbucketid, repl=True) if len(repl_servers) < obs.repl_count: print "<%s> not enough number of replication servers to observe"\ % self.__class__.__name__ obs.status = ObserveStatus.OBS_ERROR # mark out this key self._observables.put(obs.key, obs) continue if not obs.repl_servers: obs.repl_servers.update(repl_servers) self._observables.put(obs.key, obs) for server in obs.repl_servers: vals = self.obs_keys.get(server, []) vals.append(obs_key) self.obs_keys[server] = vals reqs = [] for server, keys in self.obs_keys.iteritems(): req = ObserveRequest(keys) pkt = req.pack() try: self.conns[server].s.send(pkt) except KeyError as e: print "<%s> failed to send observe pkt : %s" % ( self.__class__.__name__, e) self._add_conn(server) return None except Exception as e: print "<%s> failed to send observe pkt : %s" % ( self.__class__.__name__, e) self._refresh_conns() return None reqs.append(req) print "reqs::" print reqs return reqs
def _send(self): self.obs_keys.clear() # {server: [keys]} observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN) with self._observables.mutex: for obs in observables: vbucketid = VbucketHelper.get_vbucket_id(obs.key, self.cfg.get("vbuckets", 0)) obs_key = ObserveRequestKey(obs.key, vbucketid) if obs.persist_count > 0: persist_server = self._get_server_str(vbucketid) vals = self.obs_keys.get(persist_server, []) vals.append(obs_key) self.obs_keys[persist_server] = vals if not obs.persist_servers: obs.persist_servers.add(persist_server) self._observables.put(obs.key, obs) if obs.repl_count > 0: repl_servers = self._get_server_str(vbucketid, repl=True) if len(repl_servers) < obs.repl_count: print "<%s> not enough number of replication servers to observe"\ % self.__class__.__name__ obs.status = ObserveStatus.OBS_ERROR # mark out this key self._observables.put(obs.key, obs) continue if not obs.repl_servers: obs.repl_servers.update(repl_servers) self._observables.put(obs.key, obs) for server in obs.repl_servers: vals = self.obs_keys.get(server, []) vals.append(obs_key) self.obs_keys[server] = vals reqs = [] for server, keys in self.obs_keys.iteritems(): req = ObserveRequest(keys) pkt = req.pack() try: self.conns[server].s.send(pkt) except KeyError as e: print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e) self._add_conn(server) return None except Exception as e: print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e) self._refresh_conns() return None reqs.append(req) print "reqs::" print reqs return reqs
def observe_single(self, key, server="", timeout=0): """ send an observe command and get the response back parse the response afterwards @return (status, cas) @status -1 : network error @status -2 : protocol error @status ObserveKeyState """ cas = "" if not key: print "<%s> observe_single: invalid key" % self.__class__.__name__ return -1 vbucketid = \ VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0)) if not server: server = self._get_server_str(vbucketid) req_key = ObserveRequestKey(key, vbucketid) req = ObserveRequest([req_key]) pkt = req.pack() try: skt = self.conns[server].s except KeyError: print "<%s> observe_single: KeyError: %s" \ % (self.__class__.__name__, server) self._add_conn(server) return -1, cas try: SocketHelper.send_bytes(skt, pkt, timeout) except IOError: print "<%s> observe_single: IOError: " \ "failed to send observe pkt : %s" \ % (self.__class__.__name__, pkt) self._reconnect(self.conns[server]) self._refresh_conns() return -1, cas except socket.timeout: print "<%s> observe_single: timeout: " \ "failed to send observe pkt : %s" \ % (self.__class__.__name__, pkt) return -1, cas except Exception as e: print "<%s> observe_single: failed to send observe pkt : %s" \ % (self.__class__.__name__, e) return -1, cas try: hdr = SocketHelper.recv_bytes( skt, ObservePktFmt.OBS_RES_HDR_LEN, timeout) res = ObserveResponse() if not res.unpack_hdr(hdr): if res.status == ERR_NOT_MY_VBUCKET: self._refresh_conns() return -1, cas body = SocketHelper.recv_bytes(skt, res.body_len, timeout) res.unpack_body(body) except IOError: print "<%s> observe_single: IOError: failed to recv observe pkt" \ % self.__class__.__name__ self._reconnect(self.conns[server]) self._refresh_conns() return -1, cas except socket.timeout: print "<%s> observe_single: timeout: failed to recv observe pkt" \ % self.__class__.__name__ return -1, cas except Exception as e: print "<%s> observe_single: failed to recv observe pkt : %s" \ % (self.__class__.__name__, e) return -1, cas if not res: print "<%s> observe_single: empty response" \ % self.__class__.__name__ return -1, cas key_len = len(res.keys) if key_len != 1: # we are not supposed to receive responses for more than one key, # otherwise, it's a server side protocol error print "<%s> observe_single: invalid number of keys in response: %d"\ % (self.s.__name__, key_len) return -2, cas res_key = res.keys[0] cas = res_key.cas if res_key.key != key: print "<%s> observe_single: invalid key %s in response"\ % self.__class__.__name__ return -2, cas return res_key.key_state, cas
def observe_single(self, key, timeout=0): """ send an observe command and get the response back """ if not key: print "<%s> observe_single: invalid key" % self.__class__.__name__ return None vbucketid = VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0)) server = self._get_server_str(vbucketid) req_key = ObserveRequestKey(key, vbucketid) req = ObserveRequest([req_key]) pkt = req.pack() try: skt = self.conns[server].s except KeyError: print "<%s> KeyError: %s" % (self.__class__.__name__, server) self._add_conn(server) return None try: SocketHelper.send_bytes(skt, pkt, timeout) except IOError: print "<%s> IOError: failed to send observe pkt : %s" \ % (self.__class__.__name__, pkt) self._reconnect(self.conns[server]) self._refresh_conns() return None except socket.timeout: print "<%s> timeout: failed to send observe pkt : %s" \ % (self.__class__.__name__, pkt) return None except Exception as e: print "<%s> failed to send observe pkt : %s" \ % (self.__class__.__name__, e) return None try: hdr = SocketHelper.recv_bytes(skt, ObservePktFmt.OBS_RES_HDR_LEN, timeout) res = ObserveResponse() if not res.unpack_hdr(hdr): if res.status == ERR_NOT_MY_VBUCKET: self._refresh_conns() return None body = SocketHelper.recv_bytes(skt, res.body_len, timeout) res.unpack_body(body) except IOError: print "<%s> IOError: failed to recv observe pkt" \ % self.__class__.__name__ self._reconnect(self.conns[server]) self._refresh_conns() return None except socket.timeout: print "<%s> timeout: failed to recv observe pkt" \ % self.__class__.__name__ return None except Exception as e: print "<%s> failed to recv observe pkt : %s" \ % (self.__class__.__name__, e) return None return res