def do_watch(self): """Watch a keyspace This will conduct one watch or one read. """ try: LOG.debug("%s: pausing", self.name) try: if self.tick is None: # We have no state, so we have effectively # 'fallen off of the history' and need to # resync in full. raise etcd.EtcdEventIndexCleared() # Current versions of python-etcd use the timeout in # interesting ways. Slow URL connections + no data # to return can lead to timeouts much longer than you # might expect. So we watch for a timeout for # ourselves as well. # Here, we use both etcd Client timeout and eventlet timeout # As etcd.Client timeout is not reliable, enforce # the timeout with eventlet.Timeout # Most of the time, we timeout thanks to etcd client, # if we timeout due to eventlet, we have an ugly error message rv = None with eventlet.Timeout(self.heartbeat + 5, False): rv = self.etcd_client.watch(self.watch_path, recursive=True, index=self.tick, timeout=self.heartbeat) if rv: with self.etcd_data_lock: # The processing function is entitled to check all etcd # data. Update it before we call the processor. if rv.action == 'delete': self.etcd_data.pop(rv.key, None) else: self.etcd_data[rv.key] = rv.value # We can, in the case of a watch, hint at where the # update went. try: self.do_work(rv.action, rv.key, rv.value) except Exception: LOG.exception(('%s key %s value %s could' 'not be processed') % (rv.action, rv.key, rv.value)) # TODO(ijw) raise or not raise? This is probably # fatal and incurable, because we will only repeat # the action on the next round. raise # Update the tick only when all the above completes so that # exceptions don't cause the count to skip before the data # is processed self.tick = rv.modifiedIndex + 1 except etcd.EtcdEventIndexCleared: # We can't follow etcd history in teaspoons, so # grab the current state in its entirety. self.refresh_all_data() except (etcd.EtcdWatchTimedOut, UrllibTimeoutError): # This is normal behaviour, indicating either a watch timeout # (nothing changed) or a connection timeout (we should retry) pass
def do_watch(self): """Watch a keyspace This will conduct one watch or one read. """ try: LOG.debug("%s: pausing", self.name) try: if self.tick is None: # We have no state, so we have effectively # 'fallen off of the history' and need to # resync in full. raise etcd.EtcdEventIndexCleared() # Current versions of python-etcd use the timeout in # interesting ways. Slow URL connections + no data # to return can lead to timeouts much longer than you # might expect. So we watch for a timeout for # ourselves as well. # Here, we use both etcd Client timeout and eventlet timeout # As etcd.Client timeout is not reliable, enforce # the timeout with eventlet.Timeout # Most of the time, we timeout thanks to etcd client, # if we timeout due to eventlet, we have an ugly error message with eventlet.Timeout(self.heartbeat + 5): rv = self.etcd_client.watch(self.watch_path, recursive=True, index=self.tick, timeout=self.heartbeat) vals = [rv] next_tick = rv.modifiedIndex + 1 except etcd.EtcdEventIndexCleared: # We can't follow etcd history in teaspoons, so # grab the current state and implement it. LOG.debug("%s: resyncing in full", self.name) rv = self.etcd_client.read(self.watch_path, recursive=True) # This appears as if all the keys have been updated - # because we can't tell which have been and which haven't. vals = rv.children self.resync() next_tick = rv.etcd_index + 1 LOG.debug("%s watch index recovered: %s", self.name, str(next_tick)) for kv in vals: LOG.debug("%s: active, key %s", self.name, kv.key) try: self.do_work(kv.action, kv.key, kv.value) except Exception: LOG.exception('%s key %s value %s could not be processed' % (kv.action, kv.key, kv.value)) # TODO(ijw) raise or not raise? This is probably # fatal and incurable. raise # Update the tick only when all the above completes so that # exceptions don't cause the count to skip before the data # is processed self.tick = next_tick except (etcd.EtcdWatchTimedOut, UrllibTimeoutError, eventlet.Timeout): # This is normal behaviour, indicating either a watch timeout # (nothing changed) or a connection timeout (we should retry) pass