def start(self):
LOG.info("Start watching %s", self.prefix)
self._stopped = False
# The current etcd cluster ID.
current_cluster_id = None
while not self._stopped:
# Get the current etcdv3 cluster ID and revision, so (a) we can
# detect if the cluster ID changes, and (b) we know when to start
# watching from.
try:
cluster_id, last_revision = etcdv3.get_status()
last_revision = int(last_revision)
LOG.debug("Current cluster_id %s, revision %d", cluster_id,
last_revision)
if cluster_id != current_cluster_id:
# No particular handling here; but keep track of the
# current cluster ID and log if it changes. (In the
# circumstances that can cause a cluster ID change, our
# watch (below) for the old cluster ID would have timed out
# - either because of connection loss, or because of no
# further events coming - and then we would have looped
# back round to here; and the next watch will be created
# against the new cluster.)
if current_cluster_id is not None:
LOG.warning("Cluster ID changed")
current_cluster_id = cluster_id
except ConnectionFailedError as e:
LOG.debug("%r", e)
LOG.warning("etcd not available, will retry in 5s")
eventlet.sleep(5)
continue
# Allow subclass to do pre-snapshot processing, and to return any
# data that it will need for reconciliation after the snapshot.
my_name = self.__class__.__name__
LOG.debug("%s Calling pre-snapshot hook", my_name)
snapshot_data = self._pre_snapshot_hook()
try:
# Get all existing values and process them through the
# dispatcher.
LOG.debug("%s Loading snapshot", my_name)
for result in etcdv3.get_prefix(self.prefix,
revision=last_revision):
key, value, mod_revision = result
# Convert to what the dispatcher expects - see below.
response = Response(
action='set',
key=key,
value=value,
mod_revision=mod_revision,
)
LOG.debug("status event: %s", response)
self.dispatcher.handle_event(response)
except ConnectionFailedError as e:
LOG.debug("%r", e)
LOG.warning("etcd not available, will retry in 5s")
eventlet.sleep(5)
continue
# Allow subclass to do post-snapshot reconciliation.
LOG.debug("%s Done loading snapshot, calling post snapshot hook",
my_name)
self._post_snapshot_hook(snapshot_data)
# Now watch for any changes, starting after the revision above.
try:
# Start a watch from just after the last known revision.
LOG.debug("%s Starting to watch for updates", my_name)
event_stream, cancel = etcdv3.watch_subtree(
self.prefix, str(last_revision + 1))
# It is possible for that watch call to be affected by an etcd
# compaction, if there is a sequence of events as follows.
#
# 1. EtcdWatcher calls get_status (39 lines above) and finds
# that the etcd revision at that time is N.
#
# 2. There are at least 2 changes to the database (by any etcd
# writer, including other threads/forks of the Neutron server),
# such that the etcd revision is >= N+2, before our watch call.
#
# 3. etcd is then compacted at revision >= N+2, also before our
# watch call.
#
# 4. Our watch call then tries to create a watch starting at
# revision N+1, which is no longer available.
#
# If that happens, the etcd server sends these responses to the
# etcd3gw client, and then does NOT send any events for the
# prefix that we are monitoring:
#
# {"result":{"header":{"cluster_id":"14841639068965178418",
# "member_id":"10276657743932975437","revision":"33",
# "raft_term":"2"},"created":true}}
#
# {"result":{"header":{"cluster_id":"14841639068965178418",
# "member_id":"10276657743932975437","raft_term":"2"},
# "compact_revision":"32"}}
#
# Both of those response lines are consumed by the etcd3gw
# client/watch code, with nothing reported up to this code
# here. Hence the next thing that will happen here is timing
# out after WATCH_TIMEOUT_SECS (10s). Then we'll loop round,
# get the current revision, and start watching again from
# there.
#
# Given the things that EtcdWatcher is used for, I think that's
# good enough without more specific handling. EtcdWatcher is
# used for:
#
# - agent status, where the impacts are placing a VM on a
# compute host where Felix has died, or not using a compute
# host where Felix has just become available. For Felix
# death there is a window (TTL) of 90s anyway, so another 10s
# doesn't make a big difference.
#
# - port status, where the impact is just correct presentation
# in the OpenStack UI.
#
# - DHCP info, where the impact is dnsmasq not being able to
# answer a DHCP request. But any sensible guest OS will
# retry anyway for at least 10s, so I think we're still OK.
except Exception:
# Log and handle by restarting the loop, which means we'll get
# the tree again and then try watching again. E.g. it could be
# that the DB has just been compacted and so the revision is no
# longer available that we asked to start watching from.
LOG.exception("Exception watching status tree")
continue
# Record time of last activity on the successfully created watch.
# (This is updated below as we see watch events.)
last_event_time = monotonic_time()
def _cancel_watch_if_broken():
# Loop until we should cancel the watch, either because of
# inactivity or because of stop() having been called.
while not self._stopped:
# If WATCH_TIMEOUT_SECS has now passed since the last watch
# event, break out of this loop. If we are also writing a
# key within the tree every WATCH_TIMEOUT_SECS / 3 seconds,
# this can only happen either if there is some roundtrip
# connectivity problem, or if the watch is invalid because
# of a recent compaction. Whatever the reason, we need to
# terminate this watch and take a new overall status and
# snapshot of the tree.
time_now = monotonic_time()
if time_now > last_event_time + WATCH_TIMEOUT_SECS:
if self.round_trip_suffix is not None:
LOG.warning("Watch is not working")
else:
LOG.debug("Watch timed out")
break
if self.round_trip_suffix is not None:
# Write to a key in the tree that we are watching. If
# the watch is working normally, it will report this
# event.
etcdv3.put(self.prefix + self.round_trip_suffix,
str(time_now))
# Sleep until time for next write.
eventlet.sleep(WATCH_TIMEOUT_SECS / 3)
LOG.debug("Checked %s watch at %r", self.prefix, time_now)
# Cancel the watch
cancel()
return
# Spawn a greenlet to cancel the watch if it stops working, or if
# stop() is called. Cancelling the watch adds None to the event
# stream, so the following for loop will see that.
eventlet.spawn(_cancel_watch_if_broken)
for event in event_stream:
LOG.debug("Event: %s", event)
last_event_time = monotonic_time()
# If the EtcdWatcher has been stopped, return from the whole
# loop.
if self._stopped:
LOG.info("EtcdWatcher has been stopped")
return
# Otherwise a None event means that the watch has been
# cancelled owing to inactivity. In that case we break out
# from this loop, and the watch will be restarted.
if event is None:
LOG.debug("Watch cancelled owing to inactivity")
break
# An event at this point has a form like
#
# {u'kv': {
# u'mod_revision': u'4',
# u'value': '...',
# u'create_revision': u'4',
# u'version': u'1',
# u'key': '/calico/felix/v1/host/ubuntu-xenial...'
# }}
#
# when a key/value pair is created or updated, and like
#
# {u'type': u'DELETE',
# u'kv': {
# u'mod_revision': u'88',
# u'key': '/calico/felix/v1/host/ubuntu-xenial-...'
# }}
#
# when a key/value pair is deleted.
#
# Convert that to the form that the dispatcher expects;
# namely a response object, with:
# - response.key giving the etcd key
# - response.action being "set" or "delete"
# - whole response being passed on to the handler method.
# Handler methods here expect
# - response.key
# - response.value
key = event['kv']['key']
mod_revision = int(event['kv'].get('mod_revision', '0'))
response = Response(
action=event.get('type', 'SET').lower(),
key=key,
value=event['kv'].get('value', ''),
mod_revision=mod_revision,
)
LOG.info("Event: %s", response)
self.dispatcher.handle_event(response)
# Update last known revision.
if mod_revision > last_revision:
last_revision = mod_revision
LOG.debug("Last known revision is now %d", last_revision)
def get_all_from_etcd(self):
return etcdv3.get_prefix(datamodel_v2.subnet_dir(self.region_string))
def get_all(resource_kind, namespace,
with_labels_and_annotations=False, revision=None):
"""Read all Calico v3 resources of a certain kind from etcdv3.
- resource_kind (string): E.g. WorkloadEndpoint, Profile, etc.
- namespace (string): The namespace to get resources for.
- with_labels_and_annotations: If True, indicates to return the labels and
annotations for each resource as well as the spec.
- revision: if specified, the get is performed at the given revision
as a snapshot.
Returns a list of tuples (name, spec, mod_revision) or (name, (spec,
labels, annotations), mod_revision), one for each resource of the specified
kind, in which:
- name is the resource's name (a string)
- spec is a dict with keys as specified by the 'json:' comments in the
relevant golang struct definition (for example,
https://github.com/projectcalico/libcalico-go/blob/master/
lib/apis/v3/workloadendpoint.go#L38).
- labels is a dict containing the resource's labels
- annotations is a dict containing the resource's annotations
- mod_revision is the revision at which that resource was last modified (an
integer represented as a string).
"""
prefix = _build_key(resource_kind, namespace, '')
results = etcdv3.get_prefix(prefix, revision=revision)
tuples = []
for result in results:
key, value, mod_revision = result
name = key.split('/')[-1]
# Decode the value.
spec = labels = annotations = None
try:
value_dict = json.loads(value)
LOG.debug("value dict: %s", value_dict)
spec = value_dict['spec']
labels = value_dict['metadata'].get('labels', {})
annotations = value_dict['metadata'].get('annotations', {})
except ValueError:
# When the value is not valid JSON, we still return a tuple for
# this key, with spec, labels and annotations all as None. This is
# so that the caller can correctly differentiate between
# overwriting an existing value (which => a transaction with
# specified mod_revision) and creating a key that did not exist
# before (=> a transaction with version 0).
LOG.warning("etcd value not valid JSON (%s)", value)
if with_labels_and_annotations:
t = (name, (spec, labels, annotations), mod_revision)
else:
t = (name, spec, mod_revision)
tuples.append(t)
return tuples