def do_watch(self):
"""Watch a keyspace
This will conduct one watch or one read.
"""
try:
LOG.debug("%s: pausing", self.name)
try:
if self.tick is None:
# We have no state, so we have effectively
# 'fallen off of the history' and need to
# resync in full.
raise etcd.EtcdEventIndexCleared()
# Current versions of python-etcd use the timeout in
# interesting ways. Slow URL connections + no data
# to return can lead to timeouts much longer than you
# might expect. So we watch for a timeout for
# ourselves as well.
# Here, we use both etcd Client timeout and eventlet timeout
# As etcd.Client timeout is not reliable, enforce
# the timeout with eventlet.Timeout
# Most of the time, we timeout thanks to etcd client,
# if we timeout due to eventlet, we have an ugly error message
rv = None
with eventlet.Timeout(self.heartbeat + 5, False):
rv = self.etcd_client.watch(self.watch_path,
recursive=True,
index=self.tick,
timeout=self.heartbeat)
if rv:
with self.etcd_data_lock:
# The processing function is entitled to check all etcd
# data. Update it before we call the processor.
if rv.action == 'delete':
self.etcd_data.pop(rv.key, None)
else:
self.etcd_data[rv.key] = rv.value
# We can, in the case of a watch, hint at where the
# update went.
try:
self.do_work(rv.action, rv.key, rv.value)
except Exception:
LOG.exception(('%s key %s value %s could'
'not be processed') %
(rv.action, rv.key, rv.value))
# TODO(ijw) raise or not raise? This is probably
# fatal and incurable, because we will only repeat
# the action on the next round.
raise
# Update the tick only when all the above completes so that
# exceptions don't cause the count to skip before the data
# is processed
self.tick = rv.modifiedIndex + 1
except etcd.EtcdEventIndexCleared:
# We can't follow etcd history in teaspoons, so
# grab the current state in its entirety.
self.refresh_all_data()
except (etcd.EtcdWatchTimedOut, UrllibTimeoutError):
# This is normal behaviour, indicating either a watch timeout
# (nothing changed) or a connection timeout (we should retry)
pass
def do_watch(self):
"""Watch a keyspace
This will conduct one watch or one read.
"""
try:
LOG.debug("%s: pausing", self.name)
try:
if self.tick is None:
# We have no state, so we have effectively
# 'fallen off of the history' and need to
# resync in full.
raise etcd.EtcdEventIndexCleared()
# Current versions of python-etcd use the timeout in
# interesting ways. Slow URL connections + no data
# to return can lead to timeouts much longer than you
# might expect. So we watch for a timeout for
# ourselves as well.
# Here, we use both etcd Client timeout and eventlet timeout
# As etcd.Client timeout is not reliable, enforce
# the timeout with eventlet.Timeout
# Most of the time, we timeout thanks to etcd client,
# if we timeout due to eventlet, we have an ugly error message
with eventlet.Timeout(self.heartbeat + 5):
rv = self.etcd_client.watch(self.watch_path,
recursive=True,
index=self.tick,
timeout=self.heartbeat)
vals = [rv]
next_tick = rv.modifiedIndex + 1
except etcd.EtcdEventIndexCleared:
# We can't follow etcd history in teaspoons, so
# grab the current state and implement it.
LOG.debug("%s: resyncing in full", self.name)
rv = self.etcd_client.read(self.watch_path, recursive=True)
# This appears as if all the keys have been updated -
# because we can't tell which have been and which haven't.
vals = rv.children
self.resync()
next_tick = rv.etcd_index + 1
LOG.debug("%s watch index recovered: %s", self.name,
str(next_tick))
for kv in vals:
LOG.debug("%s: active, key %s", self.name, kv.key)
try:
self.do_work(kv.action, kv.key, kv.value)
except Exception:
LOG.exception('%s key %s value %s could not be processed' %
(kv.action, kv.key, kv.value))
# TODO(ijw) raise or not raise? This is probably
# fatal and incurable.
raise
# Update the tick only when all the above completes so that
# exceptions don't cause the count to skip before the data
# is processed
self.tick = next_tick
except (etcd.EtcdWatchTimedOut, UrllibTimeoutError, eventlet.Timeout):
# This is normal behaviour, indicating either a watch timeout
# (nothing changed) or a connection timeout (we should retry)
pass