def _cancel_watch_if_broken():
# Loop until we should cancel the watch, either because of
# inactivity or because of stop() having been called.
while not self._stopped:
# If WATCH_TIMEOUT_SECS has now passed since the last watch
# event, break out of this loop. If we are also writing a
# key within the tree every WATCH_TIMEOUT_SECS / 3 seconds,
# this can only happen either if there is some roundtrip
# connectivity problem, or if the watch is invalid because
# of a recent compaction. Whatever the reason, we need to
# terminate this watch and take a new overall status and
# snapshot of the tree.
time_now = monotonic_time()
if time_now > last_event_time + WATCH_TIMEOUT_SECS:
if self.round_trip_suffix is not None:
LOG.warning("Watch is not working")
else:
LOG.debug("Watch timed out")
break
if self.round_trip_suffix is not None:
# Write to a key in the tree that we are watching. If
# the watch is working normally, it will report this
# event.
etcdv3.put(self.prefix + self.round_trip_suffix,
str(time_now))
# Sleep until time for next write.
eventlet.sleep(WATCH_TIMEOUT_SECS / 3)
LOG.debug("Checked %s watch at %r", self.prefix, time_now)
# Cancel the watch
cancel()
return
def test_must_update(self):
# Start a real local etcd server.
self.start_etcd_server()
# Set up minimal config, so EtcdWatcher will use that etcd.
calico_config.register_options(cfg.CONF)
# Ensure etcd server is ready.
self.wait_etcd_ready()
# Try a put with MUST_UPDATE; should fail as does not yet exist.
succeeded = etcdv3.put("/testkey",
"testvalue",
mod_revision=etcdv3.MUST_UPDATE)
self.assertFalse(succeeded)
# Try a put with mod_revision 0, i.e. must create.
succeeded = etcdv3.put("/testkey", "testvalue", mod_revision=0)
self.assertTrue(succeeded)
# Try again with MUST_UPDATE; should now succeed.
succeeded = etcdv3.put("/testkey",
"testvalue2",
mod_revision=etcdv3.MUST_UPDATE)
self.assertTrue(succeeded)
# Try again with mod_revision 0; should now fail.
succeeded = etcdv3.put("/testkey", "testvalue2", mod_revision=0)
self.assertFalse(succeeded)
# Kill the etcd server.
self.stop_etcd_server()
def _write_old_key(self, lease):
# If there's a legacy election key, try to write that now too.
# Don't worry if there's a problem, as we only do this to
# assist during an upgrade.
try:
if self._old_key:
etcdv3.put(self._old_key, self.id_string, lease=lease)
except Exception as e:
self._log_exception("write old key", e)
def update_in_etcd(self, key, value, mod_revision=None):
return etcdv3.put(key, value, mod_revision=mod_revision)
def create_in_etcd(self, key, value):
return etcdv3.put(key, value, mod_revision=0)
def _become_master(self):
"""_become_master
Function to become the master. Never returns, and continually loops
updating the key as necessary.
raises: RestartElection if it fails to become master (e.g race
conditions). In this case, some other process has become
master.
Any other error from etcd is not caught in this routine.
"""
try:
ttl_lease = etcdv3.get_lease(self._ttl)
self._master = etcdv3.put(self._key,
self.id_string,
lease=ttl_lease,
mod_revision='0')
except Exception as e:
# We could be smarter about what exceptions we allow, but any kind
# of error means we should give up, and safer to have a broad
# except here. Log and reconnect.
self._log_exception("become master", e)
self._master = False
if not self._master:
LOG.info("Race: someone else beat us to be master")
raise RestartElection()
LOG.info("Successfully become master - key %s, value %s",
self._key, self.id_string)
# If there's a legacy election key, try to write that now too.
self._write_old_key(ttl_lease)
try:
while not self._stopped:
try:
LOG.debug("Refreshing master role")
# Refresh the lease.
ttl = ttl_lease.refresh()
# Also rewrite the key, so that non-masters see an event on
# the key.
if not etcdv3.put(self._key,
self.id_string,
lease=ttl_lease,
existing_value=self.id_string):
LOG.warning("Key changed or deleted; restart election")
raise RestartElection()
LOG.debug("Refreshed master role, TTL now is %d", ttl)
except RestartElection:
raise
except Exception as e:
# This is a pretty broad except statement, but anything
# going wrong means this instance gives up being the
# master.
self._log_exception("refresh master role", e)
raise RestartElection()
# If there's a legacy election key, try to write that now too.
self._write_old_key(ttl_lease)
eventlet.sleep(self._interval)
finally:
LOG.info("Exiting master refresh loop, no longer the master")
self._master = False
raise RestartElection()
def put(resource_kind, namespace, name, spec, annotations={}, labels=None,
mod_revision=None):
"""Write a Calico v3 resource to etcdv3.
- resource_kind (string): E.g. WorkloadEndpoint, Profile, etc.
- name (string): The resource's name. This is used to form its etcd key,
and also goes in its .Metadata.Name field.
- namespace (string): The namespace to put the resource in.
- spec (dict): Resource spec, as a dict with keys as specified by the
'json:' comments in the relevant golang struct definition (for example,
https://github.com/projectcalico/libcalico-go/blob/master/
lib/apis/v3/workloadendpoint.go#L38).
- annotations (dict): Annotations to set on the resource. These are merged
with existing annotations; i.e. existing annotations with other keys are
unchanged, and existing annotations with the same keys are overwritten by
these new values.
- mod_revision (string): If specified, indicates that the write should only
proceed if replacing an existing value with that mod_revision.
Returns True if the write happened successfully; False if not.
"""
key = _build_key(resource_kind, namespace, name)
value = None
try:
# Get the existing resource so we can persist its metadata.
value, _ = _get_with_metadata(resource_kind, namespace, name)
except etcdv3.KeyNotFound:
pass
except ValueError:
LOG.warning("etcd value not valid JSON, so ignoring")
if value is None:
# Build basic resource structure.
value = {
'kind': resource_kind,
'apiVersion': 'projectcalico.org/v3',
'metadata': {
'name': name,
},
}
# Ensure namespace set, for a namespaced resource.
if _is_namespaced(resource_kind):
assert namespace is not None
value['metadata']['namespace'] = namespace
# Ensure that there is a creation timestamp.
if 'creationTimestamp' not in value['metadata']:
value['metadata']['creationTimestamp'] = timestamp_now()
# Ensure that there is a UID.
if 'uid' not in value['metadata']:
value['metadata']['uid'] = str(uuid.uuid4())
# Set annotations and labels if specified. (We previously used to merge
# here, instead of overwriting, but (a) for annotations there is actually
# no use case for that, because we only use annotations on endpoints for
# which Neutron is the sole source of truth; and (b) for the use case where
# labels are used to represent security group membership it is crucial that
# we overwrite and don't merge; otherwise a VM could never be removed from
# a security group.)
if annotations:
value['metadata']['annotations'] = annotations
if labels:
value['metadata']['labels'] = labels
# Set the new spec (overriding whatever may already be there).
value['spec'] = spec
return etcdv3.put(key, json.dumps(value), mod_revision=mod_revision)