def get_profiles(self):
"""
Gets information about every profile in etcd. Returns a generator of
``Profile`` objects.
"""
LOG.info("Scanning etcd for all profiles")
try:
result = self.client.read(
PROFILE_DIR, recursive=True, timeout=ETCD_TIMEOUT
)
except etcd.EtcdKeyNotFound:
# No key yet, which is totally fine: just exit.
LOG.info("No profiles key present")
return
nodes = result.children
tag_indices = {}
rules_indices = {}
for node in nodes:
# All groups have both tags and rules, and we need the
# modifiedIndex for both.
tags_match = TAGS_KEY_RE.match(node.key)
rules_match = RULES_KEY_RE.match(node.key)
if tags_match:
profile_id = tags_match.group('profile_id')
tag_indices[profile_id] = node.modifiedIndex
elif rules_match:
profile_id = rules_match.group('profile_id')
rules_indices[profile_id] = node.modifiedIndex
else:
continue
# Check whether we have a complete set. If we do, remove them and
# yield.
if profile_id in tag_indices and profile_id in rules_indices:
tag_modified = tag_indices.pop(profile_id)
rules_modified = rules_indices.pop(profile_id)
LOG.debug("Found profile id %s", profile_id)
yield Profile(
id=profile_id,
tags_modified_index=tag_modified,
rules_modified_index=rules_modified,
tags_data=None,
rules_data=None,
)
# Quickly confirm that the tag and rule indices are empty (they should
# be).
if tag_indices or rules_indices:
LOG.warning(
"Imbalanced profile tags and rules! "
"Extra tags %s, extra rules %s", tag_indices, rules_indices
)
def resync_security_groups(self):
# Get all current security groups from the OpenStack database and key
# them on security group ID.
with self._sgs_semaphore:
self.sgs.clear()
for sg in self.driver.get_security_groups():
self.sgs[sg['id']] = sg
# As we look at the etcd data, accumulate a set of profile IDs that
# already have correct data.
correct_profiles = set()
# Read all etcd keys directly under /calico/v1/policy/profile.
try:
children = self.client.read(PROFILE_DIR, recursive=True).children
except etcd.EtcdKeyNotFound:
children = []
for child in children:
LOG.debug("etcd key: %s" % child.key)
m = TAGS_KEY_RE.match(child.key)
if m:
# If there are no policies, then read returns the top level
# node, so we need to check that this really is a profile ID.
profile_id = m.group("profile_id")
LOG.debug("Existing etcd profile data for %s" % profile_id)
try:
if profile_id in self.needed_profiles:
# This is a profile that we want. Let's read its rules and
# tags, and compare those against the current OpenStack data.
rules_key = key_for_profile_rules(profile_id)
rules = json_decoder.decode(
self.client.read(rules_key).value)
tags_key = key_for_profile_tags(profile_id)
tags = json_decoder.decode(
self.client.read(tags_key).value)
if (rules == self.profile_rules(profile_id) and
tags == self.profile_tags(profile_id)):
# The existing etcd data for this profile is completely
# correct. Remember the profile_id so that we don't
# unnecessarily write out its (unchanged) data again below.
LOG.debug("Existing etcd profile data is correct")
correct_profiles.add(profile_id)
else:
# We don't want this profile any more, so delete the key.
LOG.debug("Existing etcd profile key is now invalid")
profile_key = key_for_profile(profile_id)
self.client.delete(profile_key, recursive=True)
except etcd.EtcdKeyNotFound:
LOG.info("Etcd data appears to have been reset")
# Now write etcd data for each profile that we need and that we don't
# already know to be correct.
for profile_id in self.needed_profiles.difference(correct_profiles):
self.write_profile_to_etcd(profile_id)
def parse_if_tags(etcd_node):
m = TAGS_KEY_RE.match(etcd_node.key)
if m:
# Got some tags.
profile_id = m.group("profile_id")
if etcd_node.action == "delete":
tags = None
else:
tags = parse_tags(profile_id, etcd_node.value)
return intern(profile_id.encode("utf8")), tags
return None, None
def parse_if_tags(etcd_node):
m = TAGS_KEY_RE.match(etcd_node.key)
if m:
# Got some tags.
profile_id = m.group("profile_id")
if etcd_node.action == "delete":
tags = None
else:
tags = parse_tags(profile_id, etcd_node.value)
return intern(profile_id.encode("utf8")), tags
return None, None
def resync_security_groups(self):
# Get all current security groups from the OpenStack database and key
# them on security group ID.
self.sgs = {}
for sg in self.driver.get_security_groups():
self.sgs[sg['id']] = sg
# As we look at the etcd data, accumulate a set of profile IDs that
# already have correct data.
correct_profiles = set()
# Read all etcd keys directly under /calico/v1/policy/profile.
try:
children = self.client.read(PROFILE_DIR, recursive=True).children
except etcd.EtcdKeyNotFound:
children = []
for child in children:
LOG.debug("etcd key: %s" % child.key)
m = TAGS_KEY_RE.match(child.key)
if m:
# If there are no policies, then read returns the top level
# node, so we need to check that this really is a profile ID.
profile_id = m.group("profile_id")
LOG.debug("Existing etcd profile data for %s" % profile_id)
try:
if profile_id in self.needed_profiles:
# This is a profile that we want. Let's read its rules and
# tags, and compare those against the current OpenStack data.
rules_key = key_for_profile_rules(profile_id)
rules = json_decoder.decode(
self.client.read(rules_key).value)
tags_key = key_for_profile_tags(profile_id)
tags = json_decoder.decode(
self.client.read(tags_key).value)
if (rules == self.profile_rules(profile_id) and
tags == self.profile_tags(profile_id)):
# The existing etcd data for this profile is completely
# correct. Remember the profile_id so that we don't
# unnecessarily write out its (unchanged) data again below.
LOG.debug("Existing etcd profile data is correct")
correct_profiles.add(profile_id)
else:
# We don't want this profile any more, so delete the key.
LOG.debug("Existing etcd profile key is now invalid")
profile_key = key_for_profile(profile_id)
self.client.delete(profile_key, recursive=True)
except etcd.EtcdKeyNotFound:
LOG.info("Etcd data appears to have been reset")
# Now write etcd data for each profile that we need and that we don't
# already know to be correct.
for profile_id in self.needed_profiles.difference(correct_profiles):
self.write_profile_to_etcd(profile_id)
def parse_if_tags(etcd_node):
m = TAGS_KEY_RE.match(etcd_node.key)
if m:
# Got some tags.
profile_id = m.group("profile_id")
if etcd_node.action == "delete":
tags = None
else:
tags = json_decoder.decode(etcd_node.value)
try:
validate_tags(tags)
except ValidationFailed:
_log.exception("Validation failed for profile %s tags : %s",
profile_id, tags)
return profile_id, None
_log.debug("Found tags for profile %s : %s", profile_id, tags)
return profile_id, tags
return None, None
