def _send(self): self.obs_keys.clear() # {server: [keys]} observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN) with self._observables.mutex: for obs in observables: vbucketid = VbucketHelper.get_vbucket_id(obs.key, self.cfg.get("vbuckets", 0)) obs_key = ObserveRequestKey(obs.key, vbucketid) server = self._get_server_str(vbucketid) vals = self.obs_keys.get(server, []) vals.append(obs_key) self.obs_keys[server] = vals reqs = [] for server, keys in self.obs_keys.iteritems(): req = ObserveRequest(keys) pkt = req.pack() try: self.conns[server].s.send(pkt) except Exception as e: print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e) return None reqs.append(req) print "reqs::" print reqs return reqs
def _send(self): self.obs_keys.clear() # {server: [keys]} observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN) with self._observables.mutex: for obs in observables: vbucketid = VbucketHelper.get_vbucket_id( obs.key, self.cfg.get("vbuckets", 0)) obs_key = ObserveRequestKey(obs.key, vbucketid) if obs.persist_count > 0: persist_server = self._get_server_str(vbucketid) vals = self.obs_keys.get(persist_server, []) vals.append(obs_key) self.obs_keys[persist_server] = vals if not obs.persist_servers: obs.persist_servers.add(persist_server) self._observables.put(obs.key, obs) if obs.repl_count > 0: repl_servers = self._get_server_str(vbucketid, repl=True) if len(repl_servers) < obs.repl_count: print "<%s> not enough number of replication servers to observe"\ % self.__class__.__name__ obs.status = ObserveStatus.OBS_ERROR # mark out this key self._observables.put(obs.key, obs) continue if not obs.repl_servers: obs.repl_servers.update(repl_servers) self._observables.put(obs.key, obs) for server in obs.repl_servers: vals = self.obs_keys.get(server, []) vals.append(obs_key) self.obs_keys[server] = vals reqs = [] for server, keys in self.obs_keys.iteritems(): req = ObserveRequest(keys) pkt = req.pack() try: self.conns[server].s.send(pkt) except KeyError as e: print "<%s> failed to send observe pkt : %s" % ( self.__class__.__name__, e) self._add_conn(server) return None except Exception as e: print "<%s> failed to send observe pkt : %s" % ( self.__class__.__name__, e) self._refresh_conns() return None reqs.append(req) print "reqs::" print reqs return reqs
def _send(self): self.obs_keys.clear() # {server: [keys]} observables = self.observable_filter(ObserveStatus.OBS_UNKNOWN) with self._observables.mutex: for obs in observables: vbucketid = VbucketHelper.get_vbucket_id(obs.key, self.cfg.get("vbuckets", 0)) obs_key = ObserveRequestKey(obs.key, vbucketid) if obs.persist_count > 0: persist_server = self._get_server_str(vbucketid) vals = self.obs_keys.get(persist_server, []) vals.append(obs_key) self.obs_keys[persist_server] = vals if not obs.persist_servers: obs.persist_servers.add(persist_server) self._observables.put(obs.key, obs) if obs.repl_count > 0: repl_servers = self._get_server_str(vbucketid, repl=True) if len(repl_servers) < obs.repl_count: print "<%s> not enough number of replication servers to observe"\ % self.__class__.__name__ obs.status = ObserveStatus.OBS_ERROR # mark out this key self._observables.put(obs.key, obs) continue if not obs.repl_servers: obs.repl_servers.update(repl_servers) self._observables.put(obs.key, obs) for server in obs.repl_servers: vals = self.obs_keys.get(server, []) vals.append(obs_key) self.obs_keys[server] = vals reqs = [] for server, keys in self.obs_keys.iteritems(): req = ObserveRequest(keys) pkt = req.pack() try: self.conns[server].s.send(pkt) except KeyError as e: print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e) self._add_conn(server) return None except Exception as e: print "<%s> failed to send observe pkt : %s" % (self.__class__.__name__, e) self._refresh_conns() return None reqs.append(req) print "reqs::" print reqs return reqs
def observe_single(self, key, server="", timeout=0): """ send an observe command and get the response back parse the response afterwards @return (status, cas) @status -1 : network error @status -2 : protocol error @status ObserveKeyState """ cas = "" if not key: print "<%s> observe_single: invalid key" % self.__class__.__name__ return -1 vbucketid = \ VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0)) if not server: server = self._get_server_str(vbucketid) req_key = ObserveRequestKey(key, vbucketid) req = ObserveRequest([req_key]) pkt = req.pack() try: skt = self.conns[server].s except KeyError: print "<%s> observe_single: KeyError: %s" \ % (self.__class__.__name__, server) self._add_conn(server) return -1, cas try: SocketHelper.send_bytes(skt, pkt, timeout) except IOError: print "<%s> observe_single: IOError: " \ "failed to send observe pkt : %s" \ % (self.__class__.__name__, pkt) self._reconnect(self.conns[server]) self._refresh_conns() return -1, cas except socket.timeout: print "<%s> observe_single: timeout: " \ "failed to send observe pkt : %s" \ % (self.__class__.__name__, pkt) return -1, cas except Exception as e: print "<%s> observe_single: failed to send observe pkt : %s" \ % (self.__class__.__name__, e) return -1, cas try: hdr = SocketHelper.recv_bytes( skt, ObservePktFmt.OBS_RES_HDR_LEN, timeout) res = ObserveResponse() if not res.unpack_hdr(hdr): if res.status == ERR_NOT_MY_VBUCKET: self._refresh_conns() return -1, cas body = SocketHelper.recv_bytes(skt, res.body_len, timeout) res.unpack_body(body) except IOError: print "<%s> observe_single: IOError: failed to recv observe pkt" \ % self.__class__.__name__ self._reconnect(self.conns[server]) self._refresh_conns() return -1, cas except socket.timeout: print "<%s> observe_single: timeout: failed to recv observe pkt" \ % self.__class__.__name__ return -1, cas except Exception as e: print "<%s> observe_single: failed to recv observe pkt : %s" \ % (self.__class__.__name__, e) return -1, cas if not res: print "<%s> observe_single: empty response" \ % self.__class__.__name__ return -1, cas key_len = len(res.keys) if key_len != 1: # we are not supposed to receive responses for more than one key, # otherwise, it's a server side protocol error print "<%s> observe_single: invalid number of keys in response: %d"\ % (self.s.__name__, key_len) return -2, cas res_key = res.keys[0] cas = res_key.cas if res_key.key != key: print "<%s> observe_single: invalid key %s in response"\ % self.__class__.__name__ return -2, cas return res_key.key_state, cas
def block_for_replication(self, key, cas, num=1, timeout=0, persist=False): """ observe a key until it has been replicated to @param num of servers @param persist : block until item has been persisted to disk """ if not isinstance(num, int) or num <= 0: print "<%s> block_for_replication: invalid num %s" \ % (self.__class__.__name__, num) return False vbucketid = \ VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0)) repl_servers = self._get_server_str(vbucketid, repl=True) if persist and not self.block_for_persistence(key, cas): return False self.backoff = self.cfg.get('obs-backoff', BACKOFF) print "<%s> block_for_replication: repl_servers: %s,"\ " key: %s, cas: %s, vbucketid: %s" \ % (self.__class__.__name__, repl_servers, key, cas, vbucketid) while len(repl_servers) >= num > 0: for server in repl_servers: if num == 0: break status, new_cas = self.observe_single(key, server, timeout) if status < 0: repl_servers.remove(server) continue if new_cas and new_cas != cas: # Due to the current protocol limitations, # assume key is unique and new, skip this server repl_servers.remove(server) continue if status == ObserveKeyState.OBS_PERSISITED: num -= 1 repl_servers.remove(server) continue elif status == ObserveKeyState.OBS_FOUND: if not persist: num -= 1 repl_servers.remove(server) continue elif status == ObserveKeyState.OBS_NOT_FOUND: pass if len(repl_servers) == 1: sleep(self.backoff) self.backoff = min(self.backoff * 2, self.max_backoff) if num > 0: return False return True
def observe_single(self, key, timeout=0): """ send an observe command and get the response back """ if not key: print "<%s> observe_single: invalid key" % self.__class__.__name__ return None vbucketid = VbucketHelper.get_vbucket_id(key, self.cfg.get("vbuckets", 0)) server = self._get_server_str(vbucketid) req_key = ObserveRequestKey(key, vbucketid) req = ObserveRequest([req_key]) pkt = req.pack() try: skt = self.conns[server].s except KeyError: print "<%s> KeyError: %s" % (self.__class__.__name__, server) self._add_conn(server) return None try: SocketHelper.send_bytes(skt, pkt, timeout) except IOError: print "<%s> IOError: failed to send observe pkt : %s" \ % (self.__class__.__name__, pkt) self._reconnect(self.conns[server]) self._refresh_conns() return None except socket.timeout: print "<%s> timeout: failed to send observe pkt : %s" \ % (self.__class__.__name__, pkt) return None except Exception as e: print "<%s> failed to send observe pkt : %s" \ % (self.__class__.__name__, e) return None try: hdr = SocketHelper.recv_bytes(skt, ObservePktFmt.OBS_RES_HDR_LEN, timeout) res = ObserveResponse() if not res.unpack_hdr(hdr): if res.status == ERR_NOT_MY_VBUCKET: self._refresh_conns() return None body = SocketHelper.recv_bytes(skt, res.body_len, timeout) res.unpack_body(body) except IOError: print "<%s> IOError: failed to recv observe pkt" \ % self.__class__.__name__ self._reconnect(self.conns[server]) self._refresh_conns() return None except socket.timeout: print "<%s> timeout: failed to recv observe pkt" \ % self.__class__.__name__ return None except Exception as e: print "<%s> failed to recv observe pkt : %s" \ % (self.__class__.__name__, e) return None return res