def atomic_delete_profile(self, profile):
"""
Atomically delete a profile. This occurs in two stages: first the tag,
then the rules. Abort if the first stage fails, as we can assume that
someone else is trying to replace the profile.
This will also attempt to clean up the directory, but isn't overly
bothered if that fails.
"""
LOG.info(
"Deleting profile %s, tags modified %s, rules modified %s",
profile.id,
profile.tags_modified_index,
profile.rules_modified_index
)
self.client.delete(
key_for_profile_tags(profile.id),
prevIndex=profile.tags_modified_index,
timeout=5
)
self.client.delete(
key_for_profile_rules(profile.id),
prevIndex=profile.rules_modified_index,
timeout=5
)
# Strip the endpoint specific part of the key.
profile_key = key_for_profile(profile.id)
try:
self.client.delete(profile_key, dir=True, timeout=5)
except etcd.EtcdException as e:
LOG.debug("Failed to delete %s (%r), giving up.", profile_key, e)
def write_profile_to_etcd(self,
profile,
prev_rules_index=None,
prev_tags_index=None):
"""
Write a single security profile into etcd.
"""
LOG.debug("Writing profile %s", profile)
# python-etcd is stupid about the prevIndex keyword argument, so we
# need to explicitly filter out None-y values ourselves.
rules_kwargs = {}
if prev_rules_index is not None:
rules_kwargs['prevIndex'] = prev_rules_index
tags_kwargs = {}
if prev_tags_index is not None:
tags_kwargs['prevIndex'] = prev_tags_index
self.client.write(
key_for_profile_rules(profile.id),
json.dumps(profile_rules(profile)),
**rules_kwargs
)
self.client.write(
key_for_profile_tags(profile.id),
json.dumps(profile_tags(profile)),
**tags_kwargs
)
def get_profile_data(self, profile):
"""get_profile_data
Get data for a profile out of etcd. This should be used on profiles
returned from functions like ``get_profiles``.
:param profile: A ``Profile`` class.
:return: A ``Profile`` class with tags and rules data present.
"""
LOG.debug("Getting profile %s", profile.id)
tags_result = self.client.read(
datamodel_v1.key_for_profile_tags(profile.id), timeout=ETCD_TIMEOUT
)
rules_result = self.client.read(
datamodel_v1.key_for_profile_rules(profile.id),
timeout=ETCD_TIMEOUT
)
return Profile(
id=profile.id,
tags_modified_index=tags_result.modifiedIndex,
rules_modified_index=rules_result.modifiedIndex,
tags_data=tags_result.value,
rules_data=rules_result.value,
)
def get_profile_data(self, profile):
"""get_profile_data
Get data for a profile out of etcd. This should be used on profiles
returned from functions like ``get_profiles``.
:param profile: A ``Profile`` class.
:return: A ``Profile`` class with tags and rules data present.
"""
LOG.debug("Getting profile %s", profile.id)
tags_result = self.client.read(datamodel_v1.key_for_profile_tags(
profile.id),
timeout=ETCD_TIMEOUT)
rules_result = self.client.read(datamodel_v1.key_for_profile_rules(
profile.id),
timeout=ETCD_TIMEOUT)
return Profile(
id=profile.id,
tags_modified_index=tags_result.modifiedIndex,
rules_modified_index=rules_result.modifiedIndex,
tags_data=tags_result.value,
rules_data=rules_result.value,
)
def write_profile_to_etcd(self, profile):
"""
Write a single security profile into etcd.
"""
LOG.debug("Writing profile %s", profile)
self.client.write(key_for_profile_rules(profile.id),
json.dumps(profile_rules(profile)))
self.client.write(key_for_profile_tags(profile.id),
json.dumps(profile_tags(profile)))
def resync_security_groups(self):
# Get all current security groups from the OpenStack database and key
# them on security group ID.
with self._sgs_semaphore:
self.sgs.clear()
for sg in self.driver.get_security_groups():
self.sgs[sg['id']] = sg
# As we look at the etcd data, accumulate a set of profile IDs that
# already have correct data.
correct_profiles = set()
# Read all etcd keys directly under /calico/v1/policy/profile.
try:
children = self.client.read(PROFILE_DIR, recursive=True).children
except etcd.EtcdKeyNotFound:
children = []
for child in children:
LOG.debug("etcd key: %s" % child.key)
m = TAGS_KEY_RE.match(child.key)
if m:
# If there are no policies, then read returns the top level
# node, so we need to check that this really is a profile ID.
profile_id = m.group("profile_id")
LOG.debug("Existing etcd profile data for %s" % profile_id)
try:
if profile_id in self.needed_profiles:
# This is a profile that we want. Let's read its rules and
# tags, and compare those against the current OpenStack data.
rules_key = key_for_profile_rules(profile_id)
rules = json_decoder.decode(
self.client.read(rules_key).value)
tags_key = key_for_profile_tags(profile_id)
tags = json_decoder.decode(
self.client.read(tags_key).value)
if (rules == self.profile_rules(profile_id) and
tags == self.profile_tags(profile_id)):
# The existing etcd data for this profile is completely
# correct. Remember the profile_id so that we don't
# unnecessarily write out its (unchanged) data again below.
LOG.debug("Existing etcd profile data is correct")
correct_profiles.add(profile_id)
else:
# We don't want this profile any more, so delete the key.
LOG.debug("Existing etcd profile key is now invalid")
profile_key = key_for_profile(profile_id)
self.client.delete(profile_key, recursive=True)
except etcd.EtcdKeyNotFound:
LOG.info("Etcd data appears to have been reset")
# Now write etcd data for each profile that we need and that we don't
# already know to be correct.
for profile_id in self.needed_profiles.difference(correct_profiles):
self.write_profile_to_etcd(profile_id)
def resync_security_groups(self):
# Get all current security groups from the OpenStack database and key
# them on security group ID.
self.sgs = {}
for sg in self.driver.get_security_groups():
self.sgs[sg['id']] = sg
# As we look at the etcd data, accumulate a set of profile IDs that
# already have correct data.
correct_profiles = set()
# Read all etcd keys directly under /calico/v1/policy/profile.
try:
children = self.client.read(PROFILE_DIR, recursive=True).children
except etcd.EtcdKeyNotFound:
children = []
for child in children:
LOG.debug("etcd key: %s" % child.key)
m = TAGS_KEY_RE.match(child.key)
if m:
# If there are no policies, then read returns the top level
# node, so we need to check that this really is a profile ID.
profile_id = m.group("profile_id")
LOG.debug("Existing etcd profile data for %s" % profile_id)
try:
if profile_id in self.needed_profiles:
# This is a profile that we want. Let's read its rules and
# tags, and compare those against the current OpenStack data.
rules_key = key_for_profile_rules(profile_id)
rules = json_decoder.decode(
self.client.read(rules_key).value)
tags_key = key_for_profile_tags(profile_id)
tags = json_decoder.decode(
self.client.read(tags_key).value)
if (rules == self.profile_rules(profile_id) and
tags == self.profile_tags(profile_id)):
# The existing etcd data for this profile is completely
# correct. Remember the profile_id so that we don't
# unnecessarily write out its (unchanged) data again below.
LOG.debug("Existing etcd profile data is correct")
correct_profiles.add(profile_id)
else:
# We don't want this profile any more, so delete the key.
LOG.debug("Existing etcd profile key is now invalid")
profile_key = key_for_profile(profile_id)
self.client.delete(profile_key, recursive=True)
except etcd.EtcdKeyNotFound:
LOG.info("Etcd data appears to have been reset")
# Now write etcd data for each profile that we need and that we don't
# already know to be correct.
for profile_id in self.needed_profiles.difference(correct_profiles):
self.write_profile_to_etcd(profile_id)
def write_profile_to_etcd(self, profile):
"""
Write a single security profile into etcd.
"""
LOG.debug("Writing profile %s", profile)
self.client.write(
key_for_profile_rules(profile.id),
json.dumps(profile_rules(profile))
)
self.client.write(
key_for_profile_tags(profile.id),
json.dumps(profile_tags(profile))
)
def atomic_delete_profile(self, profile):
"""atomic_delete_profile
Atomically delete a profile. This occurs in two stages: first the tag,
then the rules. Abort if the first stage fails, as we can assume that
someone else is trying to replace the profile.
Tolerates attempting to delete keys that are already deleted.
This will also attempt to clean up the directory, but isn't overly
bothered if that fails.
"""
LOG.info(
"Deleting profile %s, tags modified %s, rules modified %s",
profile.id,
profile.tags_modified_index,
profile.rules_modified_index
)
etcd_profile_id = with_openstack_sg_prefix(profile.id)
# Try to delete tags and rules. We don't care if we can't, but we
# should log in case it's symptomatic of a wider problem.
try:
self.client.delete(
datamodel_v1.key_for_profile_tags(etcd_profile_id),
prevIndex=profile.tags_modified_index,
timeout=ETCD_TIMEOUT
)
except etcd.EtcdKeyNotFound:
LOG.info(
"Profile %s tags already deleted, nothing to do.", profile.id
)
try:
self.client.delete(
datamodel_v1.key_for_profile_rules(etcd_profile_id),
prevIndex=profile.rules_modified_index,
timeout=ETCD_TIMEOUT
)
except etcd.EtcdKeyNotFound:
LOG.info(
"Profile %s rules already deleted, nothing to do.", profile.id
)
# Strip the rules/tags specific part of the key.
profile_key = datamodel_v1.key_for_profile(etcd_profile_id)
try:
self.client.delete(profile_key, dir=True, timeout=ETCD_TIMEOUT)
except etcd.EtcdException as e:
LOG.debug("Failed to delete %s (%r), giving up.", profile_key, e)
def atomic_delete_profile(self, profile):
"""atomic_delete_profile
Atomically delete a profile. This occurs in two stages: first the tag,
then the rules. Abort if the first stage fails, as we can assume that
someone else is trying to replace the profile.
Tolerates attempting to delete keys that are already deleted.
This will also attempt to clean up the directory, but isn't overly
bothered if that fails.
"""
LOG.info(
"Deleting profile %s, tags modified %s, rules modified %s",
profile.id,
profile.tags_modified_index,
profile.rules_modified_index
)
# Try to delete tags and rules. We don't care if we can't, but we
# should log in case it's symptomatic of a wider problem.
try:
self.client.delete(
datamodel_v1.key_for_profile_tags(profile.id),
prevIndex=profile.tags_modified_index,
timeout=ETCD_TIMEOUT
)
except etcd.EtcdKeyNotFound:
LOG.info(
"Profile %s tags already deleted, nothing to do.", profile.id
)
try:
self.client.delete(
datamodel_v1.key_for_profile_rules(profile.id),
prevIndex=profile.rules_modified_index,
timeout=ETCD_TIMEOUT
)
except etcd.EtcdKeyNotFound:
LOG.info(
"Profile %s rules already deleted, nothing to do.", profile.id
)
# Strip the endpoint specific part of the key.
profile_key = datamodel_v1.key_for_profile(profile.id)
try:
self.client.delete(profile_key, dir=True, timeout=ETCD_TIMEOUT)
except etcd.EtcdException as e:
LOG.debug("Failed to delete %s (%r), giving up.", profile_key, e)
def write_profile_to_etcd(self, profile_id):
self.client.write(key_for_profile_rules(profile_id),
json.dumps(self.profile_rules(profile_id)))
self.client.write(key_for_profile_tags(profile_id),
json.dumps(self.profile_tags(profile_id)))
def write_profile_to_etcd(self, profile_id):
self.client.write(key_for_profile_rules(profile_id),
json.dumps(self.profile_rules(profile_id)))
self.client.write(key_for_profile_tags(profile_id),
json.dumps(self.profile_tags(profile_id)))
