def __init__(self, label_index):
self.label_index = label_index
self.labels_by_item_id = {}
self.labels_by_parent_id = {}
self.parent_ids_by_item_id = {}
self.item_ids_by_parent_id = MultiDict()
self._dirty_items = set()
def __init__(self):
# Cache of raw label dicts by item_id.
self.labels_by_item_id = {}
# All expressions by ID.
self.expressions_by_id = {}
# Map from expression ID to matching item_ids.
self.matches_by_expr_id = MultiDict()
self.matches_by_item_id = MultiDict()
def __init__(self):
# Cache of raw label dicts by item_id.
self.labels_by_item_id = {}
# All expressions by ID.
self.expressions_by_id = {}
# Map from expression ID to matching item_ids.
self.matches_by_expr_id = MultiDict()
self.matches_by_item_id = MultiDict()
def __init__(self, config, ip_type, iptables_updater, workload_disp_chains,
host_disp_chains, rules_manager, fip_manager,
status_reporter):
super(EndpointManager, self).__init__(qualifier=ip_type)
# Configuration and version to use
self.config = config
self.ip_type = ip_type
self.ip_version = futils.IP_TYPE_TO_VERSION[ip_type]
# Peers/utility classes.
self.iptables_updater = iptables_updater
self.workload_disp_chains = workload_disp_chains
self.host_disp_chains = host_disp_chains
self.rules_mgr = rules_manager
self.status_reporter = status_reporter
self.fip_manager = fip_manager
# All endpoint dicts that are on this host.
self.endpoints_by_id = {}
# Dict that maps from interface name ("tap1234") to endpoint ID.
self.endpoint_id_by_iface_name = {}
# Cache of IPs applied to host endpoints. (I.e. any interfaces that
# aren't workload interfaces.)
self.host_ep_ips_by_iface = {}
# Host interface dicts by ID. We'll resolve these with the IPs above
# and inject the (resolved) ones as endpoints.
self.host_eps_by_id = {}
# Cache of interfaces that we've resolved and injected as endpoints.
self.resolved_host_eps = {}
# Set of endpoints that are live on this host. I.e. ones that we've
# increffed.
self.local_endpoint_ids = set()
# Index tracking what policy applies to what endpoints.
self.policy_index = LabelValueIndex()
self.policy_index.on_match_started = self.on_policy_match_started
self.policy_index.on_match_stopped = self.on_policy_match_stopped
self._label_inherit_idx = LabelInheritanceIndex(self.policy_index)
# Tier orders by tier ID. We use this to look up the order when we're
# sorting the tiers.
self.tier_orders = {}
# Cache of the current ordering of tier IDs.
self.tier_sequence = []
# And their associated orders.
self.profile_orders = {}
# Set of profile IDs to apply to each endpoint ID.
self.pol_ids_by_ep_id = MultiDict()
self.endpoints_with_dirty_policy = set()
self._data_model_in_sync = False
self._iface_poll_greenlet = gevent.Greenlet(self._interface_poll_loop)
self._iface_poll_greenlet.link_exception(self._on_worker_died)
def __init__(self):
super(LabelValueIndex, self).__init__()
self.item_ids_by_key_value = MultiDict()
# Maps tuples of (a, b) to the set of expressions that are trivially
# satisfied by label dicts with label a = value b. For example,
# trivial expressions of the form a == "b", and a in {"b", "c", ...}
# can be evaluated by look-up in this dict.
self.literal_exprs_by_kv = MultiDict()
# Mapping from expression ID to any expressions that can't be
# represented in the way described above.
self.non_kv_expressions_by_id = {}
def __init__(self):
super(LabelValueIndex, self).__init__()
self.item_ids_by_key_value = MultiDict()
# Maps tuples of (a, b) to the set of expressions that are trivially
# satisfied by label dicts with label a = value b. For example,
# trivial expressions of the form a == "b", and a in {"b", "c", ...}
# can be evaluated by look-up in this dict.
self.literal_exprs_by_kv = MultiDict()
# Mapping from expression ID to any expressions that can't be
# represented in the way described above.
self.non_kv_expressions_by_id = {}
def __init__(self, label_index):
self.label_index = label_index
self.labels_by_item_id = {}
self.labels_by_parent_id = {}
self.parent_ids_by_item_id = {}
self.item_ids_by_parent_id = MultiDict()
self._dirty_items = set()
def __init__(self, config, ip_type,
iptables_updater,
workload_disp_chains,
host_disp_chains,
rules_manager,
fip_manager,
status_reporter):
super(EndpointManager, self).__init__(qualifier=ip_type)
# Configuration and version to use
self.config = config
self.ip_type = ip_type
self.ip_version = futils.IP_TYPE_TO_VERSION[ip_type]
# Peers/utility classes.
self.iptables_updater = iptables_updater
self.workload_disp_chains = workload_disp_chains
self.host_disp_chains = host_disp_chains
self.rules_mgr = rules_manager
self.status_reporter = status_reporter
self.fip_manager = fip_manager
# All endpoint dicts that are on this host.
self.endpoints_by_id = {}
# Dict that maps from interface name ("tap1234") to endpoint ID.
self.endpoint_id_by_iface_name = {}
# Cache of IPs applied to host endpoints. (I.e. any interfaces that
# aren't workload interfaces.)
self.host_ep_ips_by_iface = {}
# Host interface dicts by ID. We'll resolve these with the IPs above
# and inject the (resolved) ones as endpoints.
self.host_eps_by_id = {}
# Cache of interfaces that we've resolved and injected as endpoints.
self.resolved_host_eps = {}
# Set of endpoints that are live on this host. I.e. ones that we've
# increffed.
self.local_endpoint_ids = set()
# Index tracking what policy applies to what endpoints.
self.policy_index = LabelValueIndex()
self.policy_index.on_match_started = self.on_policy_match_started
self.policy_index.on_match_stopped = self.on_policy_match_stopped
self._label_inherit_idx = LabelInheritanceIndex(self.policy_index)
# Tier orders by tier ID. We use this to look up the order when we're
# sorting the tiers.
self.tier_orders = {}
# Cache of the current ordering of tier IDs.
self.tier_sequence = []
# And their associated orders.
self.profile_orders = {}
# Set of profile IDs to apply to each endpoint ID.
self.pol_ids_by_ep_id = MultiDict()
self.endpoints_with_dirty_policy = set()
self._data_model_in_sync = False
self._iface_poll_greenlet = gevent.Greenlet(self._interface_poll_loop)
self._iface_poll_greenlet.link_exception(self._on_worker_died)
def __init__(self, config, ip_type, iptables_updater, dispatch_chains,
rules_manager, fip_manager, status_reporter):
super(EndpointManager, self).__init__(qualifier=ip_type)
# Configuration and version to use
self.config = config
self.ip_type = ip_type
self.ip_version = futils.IP_TYPE_TO_VERSION[ip_type]
# Peers/utility classes.
self.iptables_updater = iptables_updater
self.dispatch_chains = dispatch_chains
self.rules_mgr = rules_manager
self.status_reporter = status_reporter
self.fip_manager = fip_manager
# All endpoint dicts that are on this host.
self.endpoints_by_id = {}
# Dict that maps from interface name ("tap1234") to endpoint ID.
self.endpoint_id_by_iface_name = {}
# Set of endpoints that are live on this host. I.e. ones that we've
# increffed.
self.local_endpoint_ids = set()
# Index tracking what policy applies to what endpoints.
self.policy_index = LabelValueIndex()
self.policy_index.on_match_started = self.on_policy_match_started
self.policy_index.on_match_stopped = self.on_policy_match_stopped
self._label_inherit_idx = LabelInheritanceIndex(self.policy_index)
# Tier orders by tier ID. We use this to look up the order when we're
# sorting the tiers.
self.tier_orders = {}
# Cache of the current ordering of tier IDs.
self.tier_sequence = []
# And their associated orders.
self.profile_orders = {}
# Set of profile IDs to apply to each endpoint ID.
self.pol_ids_by_ep_id = MultiDict()
self.endpoints_with_dirty_policy = set()
self._data_model_in_sync = False
def __init__(self, config, ip_type,
iptables_updater,
dispatch_chains,
rules_manager,
fip_manager,
status_reporter):
super(EndpointManager, self).__init__(qualifier=ip_type)
# Configuration and version to use
self.config = config
self.ip_type = ip_type
self.ip_version = futils.IP_TYPE_TO_VERSION[ip_type]
# Peers/utility classes.
self.iptables_updater = iptables_updater
self.dispatch_chains = dispatch_chains
self.rules_mgr = rules_manager
self.status_reporter = status_reporter
self.fip_manager = fip_manager
# All endpoint dicts that are on this host.
self.endpoints_by_id = {}
# Dict that maps from interface name ("tap1234") to endpoint ID.
self.endpoint_id_by_iface_name = {}
# Set of endpoints that are live on this host. I.e. ones that we've
# increffed.
self.local_endpoint_ids = set()
# Index tracking what policy applies to what endpoints.
self.policy_index = LabelValueIndex()
self.policy_index.on_match_started = self.on_policy_match_started
self.policy_index.on_match_stopped = self.on_policy_match_stopped
self._label_inherit_idx = LabelInheritanceIndex(self.policy_index)
# Tier orders by tier ID. We use this to look up the order when we're
# sorting the tiers.
self.tier_orders = {}
# Cache of the current ordering of tier IDs.
self.tier_sequence = []
# And their associated orders.
self.profile_orders = {}
# Set of profile IDs to apply to each endpoint ID.
self.pol_ids_by_ep_id = MultiDict()
self.endpoints_with_dirty_policy = set()
self._data_model_in_sync = False
def setUp(self):
super(TestMultiDict, self).setUp()
self.index = MultiDict()
class LabelInheritanceIndex(object):
"""
Wraps a LabelIndex, adding the ability for items to inherit labels
from a list of named parents.
"""
def __init__(self, label_index):
self.label_index = label_index
self.labels_by_item_id = {}
self.labels_by_parent_id = {}
self.parent_ids_by_item_id = {}
self.item_ids_by_parent_id = MultiDict()
self._dirty_items = set()
def on_item_update(self, item_id, labels_or_none, parents_or_none):
"""
Called when the labels and/or parents associated with an item are
updated.
:param item_id: opaque hashable item ID.
:param labels_or_none: Dict of labels, or None for a deletion.
:param parents_or_none: List of parents, or None for a deletion.
:return:
"""
_log.debug("Item %s updated: %s, %s", item_id,
labels_or_none, parents_or_none)
self._on_item_labels_update(item_id, labels_or_none)
self._on_item_parents_update(item_id, parents_or_none)
self._flush_updates()
def _on_item_parents_update(self, item_id, parents):
old_parents = self.parent_ids_by_item_id.get(item_id)
if old_parents != parents:
# Parents have changed. Update the index from parent ID to
# item.
if old_parents:
for parent_id in old_parents:
self.item_ids_by_parent_id.discard(parent_id, item_id)
if parents is not None:
for parent_id in parents:
self.item_ids_by_parent_id.add(parent_id, item_id)
self.parent_ids_by_item_id[item_id] = parents
else:
del self.parent_ids_by_item_id[item_id]
# Mark item dirty so that we'll re-evaluate its labels.
self._dirty_items.add(item_id)
def _on_item_labels_update(self, item_id, labels):
if self.labels_by_item_id.get(item_id) != labels:
# Labels changed, update the index and mark dirty so that we'll
# re-evaluate its labels.
if labels is not None:
self.labels_by_item_id[item_id] = labels
else:
del self.labels_by_item_id[item_id]
self._dirty_items.add(item_id)
def on_parent_labels_update(self, parent_id, labels_or_none):
"""
Called when the labels attached to a parent change.
:param parent_id: Opaque (hashable) ID of the parent.
:param labels_or_none: Dict of labels or None for a deletion.
"""
_log.debug("Parent labels for %s updated: %s", parent_id,
labels_or_none)
old_parent_labels = self.labels_by_parent_id.get(parent_id)
if old_parent_labels != labels_or_none:
# Labels changed, record the update.
if labels_or_none is not None:
self.labels_by_parent_id[parent_id] = labels_or_none
else:
del self.labels_by_parent_id[parent_id]
# Mark all the endpoints with this parent dirty.
self._dirty_items.update(
self.item_ids_by_parent_id.iter_values(parent_id)
)
self._flush_updates()
def _flush_updates(self):
_log.debug("Flushing updates...")
for item_id in self._dirty_items:
self._flush_item(item_id)
self._dirty_items.clear()
def _flush_item(self, item_id):
"""
Re-evaluates the labels for a given item ID, combining it with
its parents and updates the wrapped label index.
"""
try:
item_labels = self.labels_by_item_id[item_id]
except KeyError:
# Labels deleted, pass that through.
_log.debug("Flushing deletion of %s", item_id)
self.label_index.on_labels_update(item_id, None)
else:
# May need to combine labels with parents.
_log.debug("Combining labels for %s", item_id)
combined_labels = {}
parent_ids = self.parent_ids_by_item_id.get(item_id, [])
_log.debug("Item %s has parents %s", item_id, parent_ids)
for parent_id in parent_ids:
parent_labels = self.labels_by_parent_id.get(parent_id)
_log.debug("Parent %s has labels %s", parent_id, parent_labels)
if parent_labels:
combined_labels.update(parent_labels)
if combined_labels:
# Some contribution from parent, need to combine.
_log.debug("Combined labels: %s", combined_labels)
combined_labels.update(item_labels)
else:
# Parent makes no contribution, just use the per-item dict.
_log.debug("No parent labels, using item's dict %s",
combined_labels)
combined_labels = item_labels
self.label_index.on_labels_update(item_id, combined_labels)
class LabelValueIndex(LinearScanLabelIndex):
"""
LabelNode index that indexes the values of labels, allowing for efficient
(re)calculation of the matches for selectors of the form
'a == "b" && c == "d" && ...', which are the mainline.
"""
def __init__(self):
super(LabelValueIndex, self).__init__()
self.item_ids_by_key_value = MultiDict()
# Maps tuples of (a, b) to the set of expressions that are trivially
# satisfied by label dicts with label a = value b. For example,
# trivial expressions of the form a == "b", and a in {"b", "c", ...}
# can be evaluated by look-up in this dict.
self.literal_exprs_by_kv = MultiDict()
# Mapping from expression ID to any expressions that can't be
# represented in the way described above.
self.non_kv_expressions_by_id = {}
def on_labels_update(self, item_id, new_labels):
"""
Called to update a particular set of labels.
Triggers events for match changes.
:param item_id: an opaque (hashable) ID to associate with the
labels. There can only be one set of labels per ID.
:param new_labels: The labels dict to add to the index or None to
remove it.
"""
_log.debug("Updating labels for %s to %s", item_id, new_labels)
# Find any old labels associated with this item_id and remove the
# ones that have changed from the index.
old_labels = self.labels_by_item_id.get(item_id, {})
for k_v in old_labels.iteritems():
k, v = k_v
if new_labels is None or new_labels.get(k) != v:
_log.debug("Removing old key/value (%s, %s) from index", k, v)
self.item_ids_by_key_value.discard(k_v, item_id)
# Check all the old matches for updates. Record that we've already
# re-evaluated these expressions so we can skip them later.
seen_expr_ids = set()
old_matches = list(self.matches_by_item_id.iter_values(item_id))
for expr_id in old_matches:
seen_expr_ids.add(expr_id)
self._update_matches(expr_id, self.expressions_by_id[expr_id],
item_id, new_labels)
if new_labels is not None:
# Spin through the new labels, storing them in the index and
# looking for expressions of the form 'k == "v"', which we have
# indexed.
for k_v in new_labels.iteritems():
_log.debug("Adding (%s, %s) to index", *k_v)
self.item_ids_by_key_value.add(k_v, item_id)
for expr_id in self.literal_exprs_by_kv.iter_values(k_v):
if expr_id in seen_expr_ids:
continue
self._store_match(expr_id, item_id)
seen_expr_ids.add(expr_id)
# Spin through the remaining expressions, which we can't optimize.
for expr_id, expr in self.non_kv_expressions_by_id.iteritems():
if expr_id in seen_expr_ids:
continue
_log.debug("Checking updated labels against non-indexed expr: %s",
expr_id)
self._update_matches(expr_id, expr, item_id, new_labels)
# Finally, store the update.
self._store_labels(item_id, new_labels)
def on_expression_update(self, expr_id, expr):
"""
Called to update a particular expression.
Triggers events for match changes.
:param expr_id: an opaque (hashable) ID to associate with the
expression. There can only be one expression per ID.
:param expr: The SelectorExpression to add to the index or None to
remove it.
"""
old_expr = self.expressions_by_id.get(expr_id)
if expr == old_expr:
_log.debug("Expression %s unchanged, ignoring", expr_id)
return
# Remove any old value from the indexes. We'll then add the expression
# back in if it's suitable below.
_log.debug("Expression %s updated to %s", expr_id, expr)
if old_expr and isinstance(
old_expr.expr_op,
(LabelToLiteralEqualityNode, LabelInSetLiteralNode)):
# Either an expression of the form a == "b", or one of the form
# a in {"b", "c", ...}. Undo our index for the old entry, we'll
# then add it back in below.
label_name = old_expr.expr_op.lhs
if isinstance(old_expr.expr_op, LabelToLiteralEqualityNode):
values = [old_expr.expr_op.rhs]
else:
values = old_expr.expr_op.rhs
for value in values:
_log.debug("Old expression was indexed, removing")
k_v = label_name, value
self.literal_exprs_by_kv.discard(k_v, expr_id)
self.non_kv_expressions_by_id.pop(expr_id, None)
if not expr:
# Deletion, clean up the matches.
for item_id in list(self.matches_by_expr_id.iter_values(expr_id)):
_log.debug("Expression deleted, removing old match: %s",
item_id)
self._update_matches(expr_id, None, item_id,
self.labels_by_item_id[item_id])
elif isinstance(expr.expr_op,
(LabelToLiteralEqualityNode, LabelInSetLiteralNode)):
# Either an expression of the form a == "b", or one of the form
# a in {"b", "c", ...}. We can optimise these forms so that
# they can be evaluated by an exact lookup.
label_name = expr.expr_op.lhs
if isinstance(expr.expr_op, LabelToLiteralEqualityNode):
values = [expr.expr_op.rhs]
else:
values = expr.expr_op.rhs
# Get the old matches as a set. Then we can discard the items
# that still match, leaving us with the ones that no longer
# match.
old_matches = set(self.matches_by_expr_id.iter_values(expr_id))
for value in values:
_log.debug(
"New expression is a LabelToLiteralEqualityNode, using "
"index")
k_v = label_name, value
for item_id in self.item_ids_by_key_value.iter_values(k_v):
_log.debug("From index, %s matches %s", expr_id, item_id)
old_matches.discard(item_id)
self._store_match(expr_id, item_id)
self.literal_exprs_by_kv.add(k_v, expr_id)
# old_matches now contains only the items that this expression
# previously matched but no longer does. Remove them.
for item_id in old_matches:
_log.debug("Removing old match %s, %s", expr_id, item_id)
self._discard_match(expr_id, item_id)
else:
# The expression isn't a super-simple k == "v", let's see if we
# can still use the index...
required_kvs = expr.required_kvs if expr else None
if required_kvs:
# The expression has some required k == "v" constraints, let's
# try to find an index that reduces the work we need to do.
_log.debug("New expression requires these values: %s",
required_kvs)
best_kv = self._find_best_index(required_kvs)
# Scan over the best index that we found.
old_matches = set(self.matches_by_expr_id.iter_values(expr_id))
for item_id in self.item_ids_by_key_value.iter_values(best_kv):
old_matches.discard(item_id)
self._update_matches(expr_id, expr, item_id,
self.labels_by_item_id[item_id])
# Clean up any left-over old matches.
for item_id in old_matches:
self._update_matches(expr_id, None, item_id,
self.labels_by_item_id[item_id])
else:
# The expression was just too complex to index. Give up and
# do a linear scan.
_log.debug("%s too complex to use indexes, doing linear scan",
expr_id)
self._scan_all_labels(expr_id, expr)
self.non_kv_expressions_by_id[expr_id] = expr
# Finally, store the update.
self._store_expression(expr_id, expr)
def _find_best_index(self, required_kvs):
"""
Finds the smallest index for the given set of key/value requirements.
For example, an expression "env == 'prod' && type == 'foo'" would have
requirements [("env", "prod"), ("type", "foo")]. Suppose type=="foo"
only applies to a handful of items but env=="prod" applies to many;
this method would return ("type", "foo") as the best index.
:returns the key, value tuple for the best index to use.
"""
min_kv = None
min_num = None
for k_v in required_kvs:
num = self.item_ids_by_key_value.num_items(k_v)
if min_num is None or num < min_num:
min_kv = k_v
min_num = num
if num < 10:
# Good enough, let's get on with evaluating the
# expressions rather than spending more time looking for
# a better index.
break
_log.debug("Best index: %s, %s items", min_kv, min_num)
return min_kv
class LinearScanLabelIndex(object):
"""
A label index matches a set of SelectorExpressions against a set of
label dicts. As the matches between the two collections change,
it triggers calls to on_match_started/on_match_stopped.
LabelNode dicts are identified by their "item_id", which is an opaque
(hashable) ID for the item that the labels apply to.
Similarly, selector expressions are identified by an opaque expr_id.
A simple implementation of a label index. Every update is handled by
a full linear scan.
This class has a few purposes:
- it provides a benchmark against which other implementations can be
measured
- since it's simple, it's useful for comparative testing; any
other label index implementation should give the same end result.
- it's a base class for more advanced implementations.
"""
def __init__(self):
# Cache of raw label dicts by item_id.
self.labels_by_item_id = {}
# All expressions by ID.
self.expressions_by_id = {}
# Map from expression ID to matching item_ids.
self.matches_by_expr_id = MultiDict()
self.matches_by_item_id = MultiDict()
def on_expression_update(self, expr_id, expr):
"""
Called to update a particular expression.
Triggers events for match changes.
:param expr_id: an opaque (hashable) ID to associate with the
expression. There can only be one expression per ID.
:param expr: The SelectorExpression to add to the index or None to
remove it.
"""
_log.debug("Expression %s updated to %s", expr_id, expr)
self._scan_all_labels(expr_id, expr)
self._store_expression(expr_id, expr)
def on_labels_update(self, item_id, new_labels):
"""
Called to update a particular set of labels.
Triggers events for match changes.
:param item_id: an opaque (hashable) ID to associate with the
labels. There can only be one set of labels per ID.
:param new_labels: The labels dict to add to the index or None to
remove it.
"""
_log.debug("Labels for %s now %s", item_id, new_labels)
self._scan_all_expressions(item_id, new_labels)
self._store_labels(item_id, new_labels)
def _scan_all_labels(self, expr_id, expr):
"""
Check the given expression against all label dicts and emit
events for changes in the matching labels.
"""
_log.debug("Doing full label scan against expression %s", expr_id)
for item_id, label_values in self.labels_by_item_id.iteritems():
self._update_matches(expr_id, expr, item_id, label_values)
def _scan_all_expressions(self, item_id, new_labels):
"""
Check the given labels against all expressions and emit
events for changes in the matching labels.
"""
_log.debug("Doing full expression scan against item %s", item_id)
for expr_id, expr in self.expressions_by_id.iteritems():
self._update_matches(expr_id, expr, item_id, new_labels)
def _store_expression(self, expr_id, expr):
"""Updates expressions_by_id with the new value for an expression."""
if expr is not None:
self.expressions_by_id[expr_id] = expr
else:
self.expressions_by_id.pop(expr_id, None)
def _store_labels(self, item_id, new_labels):
"""Updates labels_by_item_id with the new labels for an item."""
if new_labels is not None:
self.labels_by_item_id[item_id] = new_labels
else:
self.labels_by_item_id.pop(item_id, None)
def _update_matches(self, expr_id, expr, item_id, label_values):
"""
(Re-)evaluates the given expression against the given labels and
stores the result.
"""
_log.debug("Re-evaluating %s against %s (%s)", expr_id, item_id,
label_values)
if expr is not None and label_values is not None:
now_matches = expr.evaluate(label_values)
_log.debug("After evaluation, now matches: %s", now_matches)
else:
_log.debug("Expr or labels missing: no match")
now_matches = False
# Update the index and generate events. These methods are idempotent
# so they'll ignore duplicate updates.
if now_matches:
self._store_match(expr_id, item_id)
else:
self._discard_match(expr_id, item_id)
def _store_match(self, expr_id, item_id):
"""
Stores that an expression matches an item.
Calls on_match_started() as a side-effect. Idempotent, does
nothing if the match is already recorded.
"""
previously_matched = self.matches_by_expr_id.contains(expr_id, item_id)
if not previously_matched:
_log.debug("%s now matches: %s", expr_id, item_id)
self.matches_by_expr_id.add(expr_id, item_id)
self.matches_by_item_id.add(item_id, expr_id)
self.on_match_started(expr_id, item_id)
def _discard_match(self, expr_id, item_id):
"""
Stores that an expression does not match an item.
Calls on_match_stopped() as a side-effect. Idempotent, does
nothing if the non-match is already recorded.
"""
previously_matched = self.matches_by_expr_id.contains(expr_id, item_id)
if previously_matched:
_log.debug("%s no longer matches %s", expr_id, item_id)
self.matches_by_expr_id.discard(expr_id, item_id)
self.matches_by_item_id.discard(item_id, expr_id)
self.on_match_stopped(expr_id, item_id)
def on_match_started(self, expr_id, item_id):
"""
Called when an expression starts matching a particular set of
labels.
Intended to be assigned/overriden.
"""
_log.debug("SelectorExpression %s now matches item %s", expr_id,
item_id)
def on_match_stopped(self, expr_id, item_id):
"""
Called when an expression stops matching a particular set of
labels.
Intended to be assigned/overriden.
"""
_log.debug("SelectorExpression %s no longer matches item %s", expr_id,
item_id)
class LinearScanLabelIndex(object):
"""
A label index matches a set of SelectorExpressions against a set of
label dicts. As the matches between the two collections change,
it triggers calls to on_match_started/on_match_stopped.
LabelNode dicts are identified by their "item_id", which is an opaque
(hashable) ID for the item that the labels apply to.
Similarly, selector expressions are identified by an opaque expr_id.
A simple implementation of a label index. Every update is handled by
a full linear scan.
This class has a few purposes:
- it provides a benchmark against which other implementations can be
measured
- since it's simple, it's useful for comparative testing; any
other label index implementation should give the same end result.
- it's a base class for more advanced implementations.
"""
def __init__(self):
# Cache of raw label dicts by item_id.
self.labels_by_item_id = {}
# All expressions by ID.
self.expressions_by_id = {}
# Map from expression ID to matching item_ids.
self.matches_by_expr_id = MultiDict()
self.matches_by_item_id = MultiDict()
def on_expression_update(self, expr_id, expr):
"""
Called to update a particular expression.
Triggers events for match changes.
:param expr_id: an opaque (hashable) ID to associate with the
expression. There can only be one expression per ID.
:param expr: The SelectorExpression to add to the index or None to
remove it.
"""
_log.debug("Expression %s updated to %s", expr_id, expr)
self._scan_all_labels(expr_id, expr)
self._store_expression(expr_id, expr)
def on_labels_update(self, item_id, new_labels):
"""
Called to update a particular set of labels.
Triggers events for match changes.
:param item_id: an opaque (hashable) ID to associate with the
labels. There can only be one set of labels per ID.
:param new_labels: The labels dict to add to the index or None to
remove it.
"""
_log.debug("Labels for %s now %s", item_id, new_labels)
self._scan_all_expressions(item_id, new_labels)
self._store_labels(item_id, new_labels)
def _scan_all_labels(self, expr_id, expr):
"""
Check the given expression against all label dicts and emit
events for changes in the matching labels.
"""
_log.debug("Doing full label scan against expression %s", expr_id)
for item_id, label_values in self.labels_by_item_id.iteritems():
self._update_matches(expr_id, expr, item_id, label_values)
def _scan_all_expressions(self, item_id, new_labels):
"""
Check the given labels against all expressions and emit
events for changes in the matching labels.
"""
_log.debug("Doing full expression scan against item %s", item_id)
for expr_id, expr in self.expressions_by_id.iteritems():
self._update_matches(expr_id, expr, item_id, new_labels)
def _store_expression(self, expr_id, expr):
"""Updates expressions_by_id with the new value for an expression."""
if expr is not None:
self.expressions_by_id[expr_id] = expr
else:
self.expressions_by_id.pop(expr_id, None)
def _store_labels(self, item_id, new_labels):
"""Updates labels_by_item_id with the new labels for an item."""
if new_labels is not None:
self.labels_by_item_id[item_id] = new_labels
else:
self.labels_by_item_id.pop(item_id, None)
def _update_matches(self, expr_id, expr, item_id, label_values):
"""
(Re-)evaluates the given expression against the given labels and
stores the result.
"""
_log.debug("Re-evaluating %s against %s (%s)", expr_id, item_id,
label_values)
if expr is not None and label_values is not None:
now_matches = expr.evaluate(label_values)
_log.debug("After evaluation, now matches: %s", now_matches)
else:
_log.debug("Expr or labels missing: no match")
now_matches = False
# Update the index and generate events. These methods are idempotent
# so they'll ignore duplicate updates.
if now_matches:
self._store_match(expr_id, item_id)
else:
self._discard_match(expr_id, item_id)
def _store_match(self, expr_id, item_id):
"""
Stores that an expression matches an item.
Calls on_match_started() as a side-effect. Idempotent, does
nothing if the match is already recorded.
"""
previously_matched = self.matches_by_expr_id.contains(expr_id, item_id)
if not previously_matched:
_log.debug("%s now matches: %s", expr_id, item_id)
self.matches_by_expr_id.add(expr_id, item_id)
self.matches_by_item_id.add(item_id, expr_id)
self.on_match_started(expr_id, item_id)
def _discard_match(self, expr_id, item_id):
"""
Stores that an expression does not match an item.
Calls on_match_stopped() as a side-effect. Idempotent, does
nothing if the non-match is already recorded.
"""
previously_matched = self.matches_by_expr_id.contains(expr_id, item_id)
if previously_matched:
_log.debug("%s no longer matches %s", expr_id, item_id)
self.matches_by_expr_id.discard(expr_id, item_id)
self.matches_by_item_id.discard(item_id, expr_id)
self.on_match_stopped(expr_id, item_id)
def on_match_started(self, expr_id, item_id):
"""
Called when an expression starts matching a particular set of
labels.
Intended to be assigned/overriden.
"""
_log.debug("SelectorExpression %s now matches item %s",
expr_id, item_id)
def on_match_stopped(self, expr_id, item_id):
"""
Called when an expression stops matching a particular set of
labels.
Intended to be assigned/overriden.
"""
_log.debug("SelectorExpression %s no longer matches item %s",
expr_id, item_id)
class LabelInheritanceIndex(object):
"""
Wraps a LabelIndex, adding the ability for items to inherit labels
from a list of named parents.
"""
def __init__(self, label_index):
self.label_index = label_index
self.labels_by_item_id = {}
self.labels_by_parent_id = {}
self.parent_ids_by_item_id = {}
self.item_ids_by_parent_id = MultiDict()
self._dirty_items = set()
def on_item_update(self, item_id, labels_or_none, parents_or_none):
"""
Called when the labels and/or parents associated with an item are
updated.
:param item_id: opaque hashable item ID.
:param labels_or_none: Dict of labels, or None for a deletion.
:param parents_or_none: List of parents, or None for a deletion.
:return:
"""
_log.debug("Item %s updated: %s, %s", item_id, labels_or_none,
parents_or_none)
self._on_item_labels_update(item_id, labels_or_none)
self._on_item_parents_update(item_id, parents_or_none)
self._flush_updates()
def _on_item_parents_update(self, item_id, parents):
old_parents = self.parent_ids_by_item_id.get(item_id)
if old_parents != parents:
# Parents have changed. Update the index from parent ID to
# item.
if old_parents:
for parent_id in old_parents:
self.item_ids_by_parent_id.discard(parent_id, item_id)
if parents is not None:
for parent_id in parents:
self.item_ids_by_parent_id.add(parent_id, item_id)
self.parent_ids_by_item_id[item_id] = parents
else:
del self.parent_ids_by_item_id[item_id]
# Mark item dirty so that we'll re-evaluate its labels.
self._dirty_items.add(item_id)
def _on_item_labels_update(self, item_id, labels):
if self.labels_by_item_id.get(item_id) != labels:
# Labels changed, update the index and mark dirty so that we'll
# re-evaluate its labels.
if labels is not None:
self.labels_by_item_id[item_id] = labels
else:
del self.labels_by_item_id[item_id]
self._dirty_items.add(item_id)
def on_parent_labels_update(self, parent_id, labels_or_none):
"""
Called when the labels attached to a parent change.
:param parent_id: Opaque (hashable) ID of the parent.
:param labels_or_none: Dict of labels or None for a deletion.
"""
_log.debug("Parent labels for %s updated: %s", parent_id,
labels_or_none)
old_parent_labels = self.labels_by_parent_id.get(parent_id)
if old_parent_labels != labels_or_none:
# Labels changed, record the update.
if labels_or_none is not None:
self.labels_by_parent_id[parent_id] = labels_or_none
else:
del self.labels_by_parent_id[parent_id]
# Mark all the endpoints with this parent dirty.
self._dirty_items.update(
self.item_ids_by_parent_id.iter_values(parent_id))
self._flush_updates()
def _flush_updates(self):
_log.debug("Flushing updates...")
for item_id in self._dirty_items:
self._flush_item(item_id)
self._dirty_items.clear()
def _flush_item(self, item_id):
"""
Re-evaluates the labels for a given item ID, combining it with
its parents and updates the wrapped label index.
"""
try:
item_labels = self.labels_by_item_id[item_id]
except KeyError:
# Labels deleted, pass that through.
_log.debug("Flushing deletion of %s", item_id)
self.label_index.on_labels_update(item_id, None)
else:
# May need to combine labels with parents.
_log.debug("Combining labels for %s", item_id)
combined_labels = {}
parent_ids = self.parent_ids_by_item_id.get(item_id, [])
_log.debug("Item %s has parents %s", item_id, parent_ids)
for parent_id in parent_ids:
parent_labels = self.labels_by_parent_id.get(parent_id)
_log.debug("Parent %s has labels %s", parent_id, parent_labels)
if parent_labels:
combined_labels.update(parent_labels)
if combined_labels:
# Some contribution from parent, need to combine.
_log.debug("Combined labels: %s", combined_labels)
combined_labels.update(item_labels)
else:
# Parent makes no contribution, just use the per-item dict.
_log.debug("No parent labels, using item's dict %s",
combined_labels)
combined_labels = item_labels
self.label_index.on_labels_update(item_id, combined_labels)
class LabelValueIndex(LinearScanLabelIndex):
"""
LabelNode index that indexes the values of labels, allowing for efficient
(re)calculation of the matches for selectors of the form
'a == "b" && c == "d" && ...', which are the mainline.
"""
def __init__(self):
super(LabelValueIndex, self).__init__()
self.item_ids_by_key_value = MultiDict()
# Maps tuples of (a, b) to the set of expressions that are trivially
# satisfied by label dicts with label a = value b. For example,
# trivial expressions of the form a == "b", and a in {"b", "c", ...}
# can be evaluated by look-up in this dict.
self.literal_exprs_by_kv = MultiDict()
# Mapping from expression ID to any expressions that can't be
# represented in the way described above.
self.non_kv_expressions_by_id = {}
def on_labels_update(self, item_id, new_labels):
"""
Called to update a particular set of labels.
Triggers events for match changes.
:param item_id: an opaque (hashable) ID to associate with the
labels. There can only be one set of labels per ID.
:param new_labels: The labels dict to add to the index or None to
remove it.
"""
_log.debug("Updating labels for %s to %s", item_id, new_labels)
# Find any old labels associated with this item_id and remove the
# ones that have changed from the index.
old_labels = self.labels_by_item_id.get(item_id, {})
for k_v in old_labels.iteritems():
k, v = k_v
if new_labels is None or new_labels.get(k) != v:
_log.debug("Removing old key/value (%s, %s) from index", k, v)
self.item_ids_by_key_value.discard(k_v, item_id)
# Check all the old matches for updates. Record that we've already
# re-evaluated these expressions so we can skip them later.
seen_expr_ids = set()
old_matches = list(self.matches_by_item_id.iter_values(item_id))
for expr_id in old_matches:
seen_expr_ids.add(expr_id)
self._update_matches(expr_id, self.expressions_by_id[expr_id],
item_id, new_labels)
if new_labels is not None:
# Spin through the new labels, storing them in the index and
# looking for expressions of the form 'k == "v"', which we have
# indexed.
for k_v in new_labels.iteritems():
_log.debug("Adding (%s, %s) to index", *k_v)
self.item_ids_by_key_value.add(k_v, item_id)
for expr_id in self.literal_exprs_by_kv.iter_values(k_v):
if expr_id in seen_expr_ids:
continue
self._store_match(expr_id, item_id)
seen_expr_ids.add(expr_id)
# Spin through the remaining expressions, which we can't optimize.
for expr_id, expr in self.non_kv_expressions_by_id.iteritems():
if expr_id in seen_expr_ids:
continue
_log.debug("Checking updated labels against non-indexed expr: %s",
expr_id)
self._update_matches(expr_id, expr, item_id, new_labels)
# Finally, store the update.
self._store_labels(item_id, new_labels)
def on_expression_update(self, expr_id, expr):
"""
Called to update a particular expression.
Triggers events for match changes.
:param expr_id: an opaque (hashable) ID to associate with the
expression. There can only be one expression per ID.
:param expr: The SelectorExpression to add to the index or None to
remove it.
"""
old_expr = self.expressions_by_id.get(expr_id)
if expr == old_expr:
_log.debug("Expression %s unchanged, ignoring", expr_id)
return
# Remove any old value from the indexes. We'll then add the expression
# back in if it's suitable below.
_log.debug("Expression %s updated to %s", expr_id, expr)
if old_expr and isinstance(old_expr.expr_op, (LabelToLiteralEqualityNode,
LabelInSetLiteralNode)):
# Either an expression of the form a == "b", or one of the form
# a in {"b", "c", ...}. Undo our index for the old entry, we'll
# then add it back in below.
label_name = old_expr.expr_op.lhs
if isinstance(old_expr.expr_op, LabelToLiteralEqualityNode):
values = [old_expr.expr_op.rhs]
else:
values = old_expr.expr_op.rhs
for value in values:
_log.debug("Old expression was indexed, removing")
k_v = label_name, value
self.literal_exprs_by_kv.discard(k_v, expr_id)
self.non_kv_expressions_by_id.pop(expr_id, None)
if not expr:
# Deletion, clean up the matches.
for item_id in list(self.matches_by_expr_id.iter_values(expr_id)):
_log.debug("Expression deleted, removing old match: %s",
item_id)
self._update_matches(expr_id, None, item_id,
self.labels_by_item_id[item_id])
elif isinstance(expr.expr_op, (LabelToLiteralEqualityNode,
LabelInSetLiteralNode)):
# Either an expression of the form a == "b", or one of the form
# a in {"b", "c", ...}. We can optimise these forms so that
# they can be evaluated by an exact lookup.
label_name = expr.expr_op.lhs
if isinstance(expr.expr_op, LabelToLiteralEqualityNode):
values = [expr.expr_op.rhs]
else:
values = expr.expr_op.rhs
# Get the old matches as a set. Then we can discard the items
# that still match, leaving us with the ones that no longer
# match.
old_matches = set(self.matches_by_expr_id.iter_values(expr_id))
for value in values:
_log.debug("New expression is a LabelToLiteralEqualityNode, using "
"index")
k_v = label_name, value
for item_id in self.item_ids_by_key_value.iter_values(k_v):
_log.debug("From index, %s matches %s", expr_id, item_id)
old_matches.discard(item_id)
self._store_match(expr_id, item_id)
self.literal_exprs_by_kv.add(k_v, expr_id)
# old_matches now contains only the items that this expression
# previously matched but no longer does. Remove them.
for item_id in old_matches:
_log.debug("Removing old match %s, %s", expr_id, item_id)
self._discard_match(expr_id, item_id)
else:
# The expression isn't a super-simple k == "v", let's see if we
# can still use the index...
required_kvs = expr.required_kvs if expr else None
if required_kvs:
# The expression has some required k == "v" constraints, let's
# try to find an index that reduces the work we need to do.
_log.debug("New expression requires these values: %s",
required_kvs)
best_kv = self._find_best_index(required_kvs)
# Scan over the best index that we found.
old_matches = set(self.matches_by_expr_id.iter_values(expr_id))
for item_id in self.item_ids_by_key_value.iter_values(best_kv):
old_matches.discard(item_id)
self._update_matches(expr_id, expr, item_id,
self.labels_by_item_id[item_id])
# Clean up any left-over old matches.
for item_id in old_matches:
self._update_matches(expr_id, None, item_id,
self.labels_by_item_id[item_id])
else:
# The expression was just too complex to index. Give up and
# do a linear scan.
_log.debug("%s too complex to use indexes, doing linear scan",
expr_id)
self._scan_all_labels(expr_id, expr)
self.non_kv_expressions_by_id[expr_id] = expr
# Finally, store the update.
self._store_expression(expr_id, expr)
def _find_best_index(self, required_kvs):
"""
Finds the smallest index for the given set of key/value requirements.
For example, an expression "env == 'prod' && type == 'foo'" would have
requirements [("env", "prod"), ("type", "foo")]. Suppose type=="foo"
only applies to a handful of items but env=="prod" applies to many;
this method would return ("type", "foo") as the best index.
:returns the key, value tuple for the best index to use.
"""
min_kv = None
min_num = None
for k_v in required_kvs:
num = self.item_ids_by_key_value.num_items(k_v)
if min_num is None or num < min_num:
min_kv = k_v
min_num = num
if num < 10:
# Good enough, let's get on with evaluating the
# expressions rather than spending more time looking for
# a better index.
break
_log.debug("Best index: %s, %s items", min_kv, min_num)
return min_kv
class EndpointManager(ReferenceManager):
def __init__(self, config, ip_type, iptables_updater, workload_disp_chains,
host_disp_chains, rules_manager, fip_manager,
status_reporter):
super(EndpointManager, self).__init__(qualifier=ip_type)
# Configuration and version to use
self.config = config
self.ip_type = ip_type
self.ip_version = futils.IP_TYPE_TO_VERSION[ip_type]
# Peers/utility classes.
self.iptables_updater = iptables_updater
self.workload_disp_chains = workload_disp_chains
self.host_disp_chains = host_disp_chains
self.rules_mgr = rules_manager
self.status_reporter = status_reporter
self.fip_manager = fip_manager
# All endpoint dicts that are on this host.
self.endpoints_by_id = {}
# Dict that maps from interface name ("tap1234") to endpoint ID.
self.endpoint_id_by_iface_name = {}
# Cache of IPs applied to host endpoints. (I.e. any interfaces that
# aren't workload interfaces.)
self.host_ep_ips_by_iface = {}
# Host interface dicts by ID. We'll resolve these with the IPs above
# and inject the (resolved) ones as endpoints.
self.host_eps_by_id = {}
# Cache of interfaces that we've resolved and injected as endpoints.
self.resolved_host_eps = {}
# Set of endpoints that are live on this host. I.e. ones that we've
# increffed.
self.local_endpoint_ids = set()
# Index tracking what policy applies to what endpoints.
self.policy_index = LabelValueIndex()
self.policy_index.on_match_started = self.on_policy_match_started
self.policy_index.on_match_stopped = self.on_policy_match_stopped
self._label_inherit_idx = LabelInheritanceIndex(self.policy_index)
# Tier orders by tier ID. We use this to look up the order when we're
# sorting the tiers.
self.tier_orders = {}
# Cache of the current ordering of tier IDs.
self.tier_sequence = []
# And their associated orders.
self.profile_orders = {}
# Set of profile IDs to apply to each endpoint ID.
self.pol_ids_by_ep_id = MultiDict()
self.endpoints_with_dirty_policy = set()
self._data_model_in_sync = False
self._iface_poll_greenlet = gevent.Greenlet(self._interface_poll_loop)
self._iface_poll_greenlet.link_exception(self._on_worker_died)
def _on_actor_started(self):
_log.info("Endpoint manager started, spawning interface poll worker.")
self._iface_poll_greenlet.start()
def _create(self, combined_id):
"""
Overrides ReferenceManager._create()
"""
if isinstance(combined_id, WloadEndpointId):
return WorkloadEndpoint(self.config, combined_id, self.ip_type,
self.iptables_updater,
self.workload_disp_chains, self.rules_mgr,
self.fip_manager, self.status_reporter)
elif isinstance(combined_id, ResolvedHostEndpointId):
return HostEndpoint(self.config, combined_id, self.ip_type,
self.iptables_updater, self.host_disp_chains,
self.rules_mgr, self.fip_manager,
self.status_reporter)
else:
raise RuntimeError("Unknown ID type: %s" % combined_id)
@actor_message()
def on_tier_data_update(self, tier, data):
"""
Message received when the metadata for a policy tier is updated
in etcd.
:param str tier: The name of the tier.
:param dict|NoneType data: The dict or None, for a deletion.
"""
_log.debug("Data for policy tier %s updated to %s", tier, data)
# Currently, the only data we care about is the order.
order = None if data is None else data["order"]
if self.tier_orders.get(tier) == order:
_log.debug("No change, ignoring")
return
if order is not None:
self.tier_orders[tier] = order
else:
del self.tier_orders[tier]
new_tier_sequence = sorted(self.tier_orders.iterkeys(),
key=lambda k: (self.tier_orders[k], k))
if self.tier_sequence != new_tier_sequence:
_log.info("Sequence of profile tiers changed, refreshing all "
"endpoints")
self.tier_sequence = new_tier_sequence
self.endpoints_with_dirty_policy.update(
self.endpoints_by_id.keys())
self._update_dirty_policy()
@actor_message()
def on_prof_labels_set(self, profile_id, labels):
_log.debug("Profile labels updated for %s: %s", profile_id, labels)
# Defer to the label index, which will call us back synchronously
# with any match changes.
self._label_inherit_idx.on_parent_labels_update(profile_id, labels)
# Process any match changes that we've recorded in the callbacks.
self._update_dirty_policy()
@actor_message()
def on_policy_selector_update(self, policy_id, selector_or_none,
order_or_none):
_log.debug("Policy %s selector updated to %s (%s)", policy_id,
selector_or_none, order_or_none)
# Defer to the label index, which will call us back synchronously
# via on_policy_match_started and on_policy_match_stopped.
self.policy_index.on_expression_update(policy_id, selector_or_none)
# Before we update the policies, check if the order has changed,
# which would mean we need to refresh all endpoints with this policy
# too.
if order_or_none != self.profile_orders.get(policy_id):
if order_or_none is not None:
self.profile_orders[policy_id] = order_or_none
else:
del self.profile_orders[policy_id]
self.endpoints_with_dirty_policy.update(
self.policy_index.matches_by_expr_id.iter_values(policy_id))
# Finally, flush any updates to our waiting endpoints.
self._update_dirty_policy()
def on_policy_match_started(self, expr_id, item_id):
"""Called by the label index when a new match is started.
Records the update but processing is deferred to
the next call to self._update_dirty_policy().
"""
_log.info("Policy %s now applies to endpoint %s", expr_id, item_id)
self.pol_ids_by_ep_id.add(item_id, expr_id)
self.endpoints_with_dirty_policy.add(item_id)
def on_policy_match_stopped(self, expr_id, item_id):
"""Called by the label index when a match stops.
Records the update but processing is deferred to
the next call to self._update_dirty_policy().
"""
_log.info("Policy %s no longer applies to endpoint %s", expr_id,
item_id)
self.pol_ids_by_ep_id.discard(item_id, expr_id)
self.endpoints_with_dirty_policy.add(item_id)
def _on_object_started(self, endpoint_id, obj):
"""
Callback from a LocalEndpoint to report that it has started.
Overrides ReferenceManager._on_object_started
"""
ep = self.endpoints_by_id.get(endpoint_id)
obj.on_endpoint_update(ep, async=True)
self._update_tiered_policy(endpoint_id)
@actor_message()
def on_datamodel_in_sync(self):
if not self._data_model_in_sync:
_log.info(
"%s: First time we've been in-sync with the datamodel,"
"sending snapshot to DispatchChains and FIPManager.", self)
self._data_model_in_sync = True
# Tell the dispatch chains about the local endpoints in advance so
# that we don't flap the dispatch chain at start-of-day. Note:
# the snapshot may contain information that is ahead of the
# state that our individual LocalEndpoint actors are sending to the
# DispatchChains actor. That is OK! The worst that can happen is
# that a LocalEndpoint undoes part of our update and then goes on
# to re-apply the update when it catches up to the snapshot.
workload_ifaces = set()
host_eps = set()
for if_name, ep_id in self.endpoint_id_by_iface_name.iteritems():
if isinstance(ep_id, WloadEndpointId):
workload_ifaces.add(if_name)
else:
host_eps.add(if_name)
self.workload_disp_chains.apply_snapshot(
frozenset(workload_ifaces), async=True)
self.host_disp_chains.apply_snapshot(frozenset(host_eps),
async=True)
self._update_dirty_policy()
nat_maps = {}
for ep_id, ep in self.endpoints_by_id.iteritems():
if ep_id in self.local_endpoint_ids:
nat_map = ep.get(nat_key(self.ip_type), None)
if nat_map:
nat_maps[ep_id] = nat_map
self.fip_manager.apply_snapshot(nat_maps, async=True)
@actor_message()
def on_host_ep_update(self, combined_id, data):
if combined_id.host != self.config.HOSTNAME:
_log.debug("Skipping endpoint %s; not on our host.", combined_id)
return
if data is not None:
self.host_eps_by_id[combined_id] = data
else:
self.host_eps_by_id.pop(combined_id, None)
self._resolve_host_eps()
@actor_message()
def on_endpoint_update(self, endpoint_id, endpoint, force_reprogram=False):
"""
Event to indicate that an endpoint has been updated (including
creation or deletion).
:param EndpointId endpoint_id: The endpoint ID in question.
:param dict[str]|NoneType endpoint: Dictionary of all endpoint
data or None if the endpoint is to be deleted.
"""
if endpoint_id.host != self.config.HOSTNAME:
_log.debug("Skipping endpoint %s; not on our host.", endpoint_id)
return
old_ep = self.endpoints_by_id.get(endpoint_id, {})
old_iface_name = old_ep.get("name")
new_iface_name = (endpoint or {}).get("name")
if (old_iface_name is not None and new_iface_name is not None
and old_iface_name != new_iface_name):
# Special-case: if the interface name of an active endpoint
# changes we need to clean up routes and iptables and start from
# scratch. Force that through the deletion path so that we don't
# introduce any more complexity in LocalEndpoint.
_log.info(
"Name of interface for endpoint %s changed from %s "
"to %s. Forcing a delete/re-add.", endpoint_id,
old_iface_name, new_iface_name)
self._on_endpoint_update_internal(endpoint_id, None,
force_reprogram)
self._on_endpoint_update_internal(endpoint_id, endpoint,
force_reprogram)
def _on_endpoint_update_internal(self,
endpoint_id,
endpoint,
force_reprogram=False):
"""Handles a single update or deletion of an endpoint.
Increfs/decrefs the actor as appropriate and forwards on the update
if the endpoint is active.
:param EndpointId endpoint_id: The endpoint ID in question.
:param dict[str]|NoneType endpoint: Dictionary of all endpoint
data or None if the endpoint is to be deleted.
"""
if self._is_starting_or_live(endpoint_id):
# Local endpoint thread is running; tell it of the change.
_log.info("Update for live endpoint %s", endpoint_id)
self.objects_by_id[endpoint_id].on_endpoint_update(
endpoint, force_reprogram=force_reprogram, async=True)
old_ep = self.endpoints_by_id.pop(endpoint_id, {})
# Interface name shouldn't change but popping it now is correct for
# deletes and we add it back in below on create/modify.
old_iface_name = old_ep.get("name")
self.endpoint_id_by_iface_name.pop(old_iface_name, None)
if endpoint is None:
# Deletion. Remove from the list.
_log.info("Endpoint %s deleted", endpoint_id)
if endpoint_id in self.local_endpoint_ids:
self.decref(endpoint_id)
self.local_endpoint_ids.remove(endpoint_id)
self._label_inherit_idx.on_item_update(endpoint_id, None, None)
assert endpoint_id not in self.pol_ids_by_ep_id
else:
# Creation or modification
_log.info("Endpoint %s modified or created", endpoint_id)
self.endpoints_by_id[endpoint_id] = endpoint
self.endpoint_id_by_iface_name[endpoint["name"]] = endpoint_id
if endpoint_id not in self.local_endpoint_ids:
# This will trigger _on_object_activated to pass the endpoint
# we just saved off to the endpoint.
_log.debug("Endpoint wasn't known before, increffing it")
self.local_endpoint_ids.add(endpoint_id)
self.get_and_incref(endpoint_id)
self._label_inherit_idx.on_item_update(
endpoint_id, endpoint.get("labels", {}),
endpoint.get("profile_ids", []))
self._update_dirty_policy()
@actor_message()
def on_interface_update(self, name, iface_up):
"""
Called when an interface is created or changes state.
The interface may be any interface on the host, not necessarily
one managed by any endpoint of this server.
"""
try:
endpoint_id = self.endpoint_id_by_iface_name[name]
except KeyError:
_log.debug("Update on interface %s that we do not care about",
name)
else:
_log.info("Endpoint %s received interface update for %s",
endpoint_id, name)
if self._is_starting_or_live(endpoint_id):
# LocalEndpoint is running, so tell it about the change.
ep = self.objects_by_id[endpoint_id]
ep.on_interface_update(iface_up, async=True)
def _interface_poll_loop(self):
"""Greenlet: Polls host endpoints for changes to their IP addresses.
Sends updates to the EndpointManager via the _on_iface_ips_update()
message.
If polling is disabled, then it reads the interfaces once and then
stops.
"""
known_interfaces = {}
while True:
known_interfaces = self._poll_interfaces(known_interfaces)
if self.config.HOST_IF_POLL_INTERVAL_SECS <= 0:
_log.info("Host interface polling disabled, stopping after "
"initial read. Further changes to host endpoint "
"IPs will be ignored.")
break
gevent.sleep(self.config.HOST_IF_POLL_INTERVAL_SECS)
def _poll_interfaces(self, known_interfaces):
"""Does a single poll of the host interfaces, looking for IP changes.
Sends updates to the EndpointManager via the _on_iface_ips_update()
message.
This is broken out form the loop above to make it easier to test.
:param known_interfaces:
:return:
"""
# We only care about host interfaces, not workload ones.
exclude_prefixes = self.config.IFACE_PREFIX
# Get the IPs for each interface.
ips_by_iface = devices.list_ips_by_iface(self.ip_type)
for iface, ips in ips_by_iface.items():
ignore_iface = any(
iface.startswith(prefix) for prefix in exclude_prefixes)
if ignore_iface:
# Ignore non-host interfaces.
ips_by_iface.pop(iface)
else:
# Compare with the set of IPs that were there before.
# We pop interfaces that we see so that we can clean up
# deletions below.
old_ips = known_interfaces.pop(iface, None)
if old_ips != ips:
_log.debug("IPs of interface %s changed to %s", iface, ips)
self._on_iface_ips_update(iface, ips, async=True)
# Clean up deletions. Anything left in known_interfaces has
# been deleted.
for iface, ips in known_interfaces.iteritems():
self._on_iface_ips_update(iface, None, async=True)
# Update our cache of known interfaces for the next loop.
return ips_by_iface
@actor_message()
def _on_iface_ips_update(self, iface_name, ip_addrs):
"""Message sent by _poll_interface_ips when it detects a change.
:param iface_name: Name of the interface that has been updated.
:param ip_addrs: set of IP addresses, or None if the interface no
longer exists (or has no IPs).
"""
_log.info("Interface %s now has IPs %s", iface_name, ip_addrs)
if ip_addrs is not None:
self.host_ep_ips_by_iface[iface_name] = ip_addrs
else:
self.host_ep_ips_by_iface.pop(iface_name, None)
# Since changes to IPs can change which host endpoint objects apply to
# which interfaces, we need to resolve IPs and host endpoints.
self._resolve_host_eps()
def _resolve_host_eps(self):
"""Resolves the host endpoint data we've learned from etcd with
IP addresses and interface names learned from the kernel.
Host interfaces that have matching IPs get combined with interface
name learned from the kernel and updated via on_endpoint_update().
In the case where multiple interfaces have the same IP address,
a copy of the host endpoint will be resolved with each interface.
"""
# Invert the interface name to IP mapping to allow us to do an IP to
# interface name lookup.
iface_names_by_ip = defaultdict(set)
for iface, ips in self.host_ep_ips_by_iface.iteritems():
for ip in ips:
iface_names_by_ip[ip].add(iface)
# Iterate over the host endpoints, looking for corresponding IPs.
resolved_ifaces = {}
iface_name_to_id = {}
# For repeatability, we sort the endpoint data. We don't care what
# the sort order is, only that it's stable so we just use the repr()
# of the ID.
for combined_id, host_ep in sorted(self.host_eps_by_id.iteritems(),
key=lambda h: repr(h[0])):
addrs_key = "expected_ipv%s_addrs" % self.ip_version
if "name" in host_ep:
# This interface has an explicit name in the data so it's
# already resolved.
resolved_id = combined_id.resolve(host_ep["name"])
resolved_ifaces[resolved_id] = host_ep
elif addrs_key in host_ep:
# No explicit name, look for an interface with a matching IP.
expected_ips = IPSet(host_ep[addrs_key])
for ip, iface_names in sorted(iface_names_by_ip.iteritems()):
if ip in expected_ips:
# This endpoint matches the IP, loop over the (usually
# one) interface with that IP. Sort the names to avoid
# non-deterministic behaviour if there are multiple
# conflicting matches.
_log.debug("Host endpoint %s matches interfaces: %s",
combined_id, iface_names)
for iface_name in sorted(iface_names):
# Check for conflicting matches.
prev_match = iface_name_to_id.get(iface_name)
if prev_match == combined_id:
# Already matched this interface by a different
# IP address.
continue
elif prev_match is not None:
# Already matched a different interface.
# First match wins.
_log.warn(
"Interface %s matched with multiple "
"entries in datamodel; using %s",
iface_name, prev_match)
continue
else:
# Else, this is the first match, record it.
iface_name_to_id[iface_name] = combined_id
# Got a match. Since it's possible to match
# multiple interfaces by IP, we add the interface
# name into the ID to disambiguate.
resolved_id = combined_id.resolve(iface_name)
resolved_data = host_ep.copy()
resolved_data["name"] = iface_name
resolved_ifaces[resolved_id] = resolved_data
# Fire in deletions for interfaces that no longer resolve.
for resolved_id in self.resolved_host_eps.keys():
if resolved_id not in resolved_ifaces:
_log.debug("%s no longer matches", resolved_id)
self.on_endpoint_update(resolved_id, None)
# Fire in the updates for the new data.
for resolved_id, data in resolved_ifaces.iteritems():
if self.resolved_host_eps.get(resolved_id) != data:
_log.debug("Updating data for %s", resolved_id)
self.on_endpoint_update(resolved_id, data)
# Update the cache so we can calculate deltas next time.
self.resolved_host_eps = resolved_ifaces
def _update_dirty_policy(self):
if not self._data_model_in_sync:
_log.debug("Datamodel not in sync, postponing update to policy")
return
_log.debug("Endpoints with dirty policy: %s",
self.endpoints_with_dirty_policy)
while self.endpoints_with_dirty_policy:
ep_id = self.endpoints_with_dirty_policy.pop()
if self._is_starting_or_live(ep_id):
self._update_tiered_policy(ep_id)
def _update_tiered_policy(self, ep_id):
"""
Sends an updated list of tiered policy to an endpoint.
Recalculates the list.
:param ep_id: ID of the endpoint to send an update to.
"""
_log.debug("Updating policies for %s from %s", ep_id,
self.pol_ids_by_ep_id)
# Order the profiles by tier and profile order, using the name of the
# tier and profile as a tie-breaker if the orders are the same.
profiles = []
for pol_id in self.pol_ids_by_ep_id.iter_values(ep_id):
try:
tier_order = self.tier_orders[pol_id.tier]
except KeyError:
_log.warn(
"Ignoring policy %s because its tier metadata is "
"missing.", pol_id)
continue
profile_order = self.profile_orders[pol_id]
profiles.append((tier_order, pol_id.tier, profile_order,
pol_id.policy_id, pol_id))
profiles.sort()
# Convert to an ordered dict from tier to list of profiles.
pols_by_tier = OrderedDict()
for _, tier, _, _, pol_id in profiles:
pols_by_tier.setdefault(tier, []).append(pol_id)
endpoint = self.objects_by_id[ep_id]
endpoint.on_tiered_policy_update(pols_by_tier, async=True)
def _on_worker_died(self, watch_greenlet):
"""
Greenlet: spawned by the gevent Hub if our worker thread dies.
"""
_log.critical("Worker greenlet died: %s; exiting.", watch_greenlet)
sys.exit(1)
def setUp(self):
super(TestMultiDict, self).setUp()
self.index = MultiDict()
class EndpointManager(ReferenceManager):
def __init__(self, config, ip_type,
iptables_updater,
workload_disp_chains,
host_disp_chains,
rules_manager,
fip_manager,
status_reporter):
super(EndpointManager, self).__init__(qualifier=ip_type)
# Configuration and version to use
self.config = config
self.ip_type = ip_type
self.ip_version = futils.IP_TYPE_TO_VERSION[ip_type]
# Peers/utility classes.
self.iptables_updater = iptables_updater
self.workload_disp_chains = workload_disp_chains
self.host_disp_chains = host_disp_chains
self.rules_mgr = rules_manager
self.status_reporter = status_reporter
self.fip_manager = fip_manager
# All endpoint dicts that are on this host.
self.endpoints_by_id = {}
# Dict that maps from interface name ("tap1234") to endpoint ID.
self.endpoint_id_by_iface_name = {}
# Cache of IPs applied to host endpoints. (I.e. any interfaces that
# aren't workload interfaces.)
self.host_ep_ips_by_iface = {}
# Host interface dicts by ID. We'll resolve these with the IPs above
# and inject the (resolved) ones as endpoints.
self.host_eps_by_id = {}
# Cache of interfaces that we've resolved and injected as endpoints.
self.resolved_host_eps = {}
# Set of endpoints that are live on this host. I.e. ones that we've
# increffed.
self.local_endpoint_ids = set()
# Index tracking what policy applies to what endpoints.
self.policy_index = LabelValueIndex()
self.policy_index.on_match_started = self.on_policy_match_started
self.policy_index.on_match_stopped = self.on_policy_match_stopped
self._label_inherit_idx = LabelInheritanceIndex(self.policy_index)
# Tier orders by tier ID. We use this to look up the order when we're
# sorting the tiers.
self.tier_orders = {}
# Cache of the current ordering of tier IDs.
self.tier_sequence = []
# And their associated orders.
self.profile_orders = {}
# Set of profile IDs to apply to each endpoint ID.
self.pol_ids_by_ep_id = MultiDict()
self.endpoints_with_dirty_policy = set()
self._data_model_in_sync = False
self._iface_poll_greenlet = gevent.Greenlet(self._interface_poll_loop)
self._iface_poll_greenlet.link_exception(self._on_worker_died)
def _on_actor_started(self):
_log.info("Endpoint manager started, spawning interface poll worker.")
self._iface_poll_greenlet.start()
def _create(self, combined_id):
"""
Overrides ReferenceManager._create()
"""
if isinstance(combined_id, WloadEndpointId):
return WorkloadEndpoint(self.config,
combined_id,
self.ip_type,
self.iptables_updater,
self.workload_disp_chains,
self.rules_mgr,
self.fip_manager,
self.status_reporter)
elif isinstance(combined_id, ResolvedHostEndpointId):
return HostEndpoint(self.config,
combined_id,
self.ip_type,
self.iptables_updater,
self.host_disp_chains,
self.rules_mgr,
self.fip_manager,
self.status_reporter)
else:
raise RuntimeError("Unknown ID type: %s" % combined_id)
@actor_message()
def on_tier_data_update(self, tier, data):
"""
Message received when the metadata for a policy tier is updated
in etcd.
:param str tier: The name of the tier.
:param dict|NoneType data: The dict or None, for a deletion.
"""
_log.debug("Data for policy tier %s updated to %s", tier, data)
# Currently, the only data we care about is the order.
order = None if data is None else data["order"]
if self.tier_orders.get(tier) == order:
_log.debug("No change, ignoring")
return
if order is not None:
self.tier_orders[tier] = order
else:
del self.tier_orders[tier]
new_tier_sequence = sorted(self.tier_orders.iterkeys(),
key=lambda k: (self.tier_orders[k], k))
if self.tier_sequence != new_tier_sequence:
_log.info("Sequence of profile tiers changed, refreshing all "
"endpoints")
self.tier_sequence = new_tier_sequence
self.endpoints_with_dirty_policy.update(
self.endpoints_by_id.keys()
)
self._update_dirty_policy()
@actor_message()
def on_prof_labels_set(self, profile_id, labels):
_log.debug("Profile labels updated for %s: %s", profile_id, labels)
# Defer to the label index, which will call us back synchronously
# with any match changes.
self._label_inherit_idx.on_parent_labels_update(profile_id, labels)
# Process any match changes that we've recorded in the callbacks.
self._update_dirty_policy()
@actor_message()
def on_policy_selector_update(self, policy_id, selector_or_none,
order_or_none):
_log.debug("Policy %s selector updated to %s (%s)", policy_id,
selector_or_none, order_or_none)
# Defer to the label index, which will call us back synchronously
# via on_policy_match_started and on_policy_match_stopped.
self.policy_index.on_expression_update(policy_id,
selector_or_none)
# Before we update the policies, check if the order has changed,
# which would mean we need to refresh all endpoints with this policy
# too.
if order_or_none != self.profile_orders.get(policy_id):
if order_or_none is not None:
self.profile_orders[policy_id] = order_or_none
else:
del self.profile_orders[policy_id]
self.endpoints_with_dirty_policy.update(
self.policy_index.matches_by_expr_id.iter_values(policy_id)
)
# Finally, flush any updates to our waiting endpoints.
self._update_dirty_policy()
def on_policy_match_started(self, expr_id, item_id):
"""Called by the label index when a new match is started.
Records the update but processing is deferred to
the next call to self._update_dirty_policy().
"""
_log.info("Policy %s now applies to endpoint %s", expr_id, item_id)
self.pol_ids_by_ep_id.add(item_id, expr_id)
self.endpoints_with_dirty_policy.add(item_id)
def on_policy_match_stopped(self, expr_id, item_id):
"""Called by the label index when a match stops.
Records the update but processing is deferred to
the next call to self._update_dirty_policy().
"""
_log.info("Policy %s no longer applies to endpoint %s",
expr_id, item_id)
self.pol_ids_by_ep_id.discard(item_id, expr_id)
self.endpoints_with_dirty_policy.add(item_id)
def _on_object_started(self, endpoint_id, obj):
"""
Callback from a LocalEndpoint to report that it has started.
Overrides ReferenceManager._on_object_started
"""
ep = self.endpoints_by_id.get(endpoint_id)
obj.on_endpoint_update(ep, async=True)
self._update_tiered_policy(endpoint_id)
@actor_message()
def on_datamodel_in_sync(self):
if not self._data_model_in_sync:
_log.info("%s: First time we've been in-sync with the datamodel,"
"sending snapshot to DispatchChains and FIPManager.",
self)
self._data_model_in_sync = True
# Tell the dispatch chains about the local endpoints in advance so
# that we don't flap the dispatch chain at start-of-day. Note:
# the snapshot may contain information that is ahead of the
# state that our individual LocalEndpoint actors are sending to the
# DispatchChains actor. That is OK! The worst that can happen is
# that a LocalEndpoint undoes part of our update and then goes on
# to re-apply the update when it catches up to the snapshot.
workload_ifaces = set()
host_eps = set()
for if_name, ep_id in self.endpoint_id_by_iface_name.iteritems():
if isinstance(ep_id, WloadEndpointId):
workload_ifaces.add(if_name)
else:
host_eps.add(if_name)
self.workload_disp_chains.apply_snapshot(
frozenset(workload_ifaces), async=True
)
self.host_disp_chains.apply_snapshot(
frozenset(host_eps), async=True
)
self._update_dirty_policy()
nat_maps = {}
for ep_id, ep in self.endpoints_by_id.iteritems():
if ep_id in self.local_endpoint_ids:
nat_map = ep.get(nat_key(self.ip_type), None)
if nat_map:
nat_maps[ep_id] = nat_map
self.fip_manager.apply_snapshot(nat_maps, async=True)
@actor_message()
def on_host_ep_update(self, combined_id, data):
if data is not None:
self.host_eps_by_id[combined_id] = data
else:
self.host_eps_by_id.pop(combined_id, None)
self._resolve_host_eps()
@actor_message()
def on_endpoint_update(self, endpoint_id, endpoint, force_reprogram=False):
"""
Event to indicate that an endpoint has been updated (including
creation or deletion).
:param EndpointId endpoint_id: The endpoint ID in question.
:param dict[str]|NoneType endpoint: Dictionary of all endpoint
data or None if the endpoint is to be deleted.
"""
if endpoint_id.host != self.config.HOSTNAME:
_log.debug("Skipping endpoint %s; not on our host.", endpoint_id)
return
if self._is_starting_or_live(endpoint_id):
# Local endpoint thread is running; tell it of the change.
_log.info("Update for live endpoint %s", endpoint_id)
self.objects_by_id[endpoint_id].on_endpoint_update(
endpoint, force_reprogram=force_reprogram, async=True)
old_ep = self.endpoints_by_id.pop(endpoint_id, {})
# Interface name shouldn't change but popping it now is correct for
# deletes and we add it back in below on create/modify.
old_iface_name = old_ep.get("name")
self.endpoint_id_by_iface_name.pop(old_iface_name, None)
if endpoint is None:
# Deletion. Remove from the list.
_log.info("Endpoint %s deleted", endpoint_id)
if endpoint_id in self.local_endpoint_ids:
self.decref(endpoint_id)
self.local_endpoint_ids.remove(endpoint_id)
self._label_inherit_idx.on_item_update(endpoint_id, None, None)
assert endpoint_id not in self.pol_ids_by_ep_id
else:
# Creation or modification
_log.info("Endpoint %s modified or created", endpoint_id)
self.endpoints_by_id[endpoint_id] = endpoint
self.endpoint_id_by_iface_name[endpoint["name"]] = endpoint_id
if endpoint_id not in self.local_endpoint_ids:
# This will trigger _on_object_activated to pass the endpoint
# we just saved off to the endpoint.
_log.debug("Endpoint wasn't known before, increffing it")
self.local_endpoint_ids.add(endpoint_id)
self.get_and_incref(endpoint_id)
self._label_inherit_idx.on_item_update(
endpoint_id,
endpoint.get("labels", {}),
endpoint.get("profile_ids", [])
)
self._update_dirty_policy()
@actor_message()
def on_interface_update(self, name, iface_up):
"""
Called when an interface is created or changes state.
The interface may be any interface on the host, not necessarily
one managed by any endpoint of this server.
"""
try:
endpoint_id = self.endpoint_id_by_iface_name[name]
except KeyError:
_log.debug("Update on interface %s that we do not care about",
name)
else:
_log.info("Endpoint %s received interface update for %s",
endpoint_id, name)
if self._is_starting_or_live(endpoint_id):
# LocalEndpoint is running, so tell it about the change.
ep = self.objects_by_id[endpoint_id]
ep.on_interface_update(iface_up, async=True)
def _interface_poll_loop(self):
"""Greenlet: Polls host endpoints for changes to their IP addresses.
Sends updates to the EndpointManager via the _on_iface_ips_update()
message.
If polling is disabled, then it reads the interfaces once and then
stops.
"""
known_interfaces = {}
while True:
known_interfaces = self._poll_interfaces(known_interfaces)
if self.config.HOST_IF_POLL_INTERVAL_SECS <= 0:
_log.info("Host interface polling disabled, stopping after "
"initial read. Further changes to host endpoint "
"IPs will be ignored.")
break
gevent.sleep(self.config.HOST_IF_POLL_INTERVAL_SECS)
def _poll_interfaces(self, known_interfaces):
"""Does a single poll of the host interfaces, looking for IP changes.
Sends updates to the EndpointManager via the _on_iface_ips_update()
message.
This is broken out form the loop above to make it easier to test.
:param known_interfaces:
:return:
"""
# We only care about host interfaces, not workload ones.
exclude_prefix = self.config.IFACE_PREFIX
# Get the IPs for each interface.
ips_by_iface = devices.list_ips_by_iface(self.ip_type)
for iface, ips in ips_by_iface.items():
if iface.startswith(exclude_prefix):
# Ignore non-host interfaces.
ips_by_iface.pop(iface)
else:
# Compare with the set of IPs that were there before.
# We pop interfaces that we see so that we can clean up
# deletions below.
old_ips = known_interfaces.pop(iface, None)
if old_ips != ips:
_log.debug("IPs of interface %s changed to %s",
iface, ips)
self._on_iface_ips_update(iface, ips, async=True)
# Clean up deletions. Anything left in known_interfaces has
# been deleted.
for iface, ips in known_interfaces.iteritems():
self._on_iface_ips_update(iface, None, async=True)
# Update our cache of known interfaces for the next loop.
return ips_by_iface
@actor_message()
def _on_iface_ips_update(self, iface_name, ip_addrs):
"""Message sent by _poll_interface_ips when it detects a change.
:param iface_name: Name of the interface that has been updated.
:param ip_addrs: set of IP addresses, or None if the interface no
longer exists (or has no IPs).
"""
_log.info("Interface %s now has IPs %s", iface_name, ip_addrs)
if ip_addrs is not None:
self.host_ep_ips_by_iface[iface_name] = ip_addrs
else:
self.host_ep_ips_by_iface.pop(iface_name, None)
# Since changes to IPs can change which host endpoint objects apply to
# which interfaces, we need to resolve IPs and host endpoints.
self._resolve_host_eps()
def _resolve_host_eps(self):
"""Resolves the host endpoint data we've learned from etcd with
IP addresses and interface names learned from the kernel.
Host interfaces that have matching IPs get combined with interface
name learned from the kernel and updated via on_endpoint_update().
In the case where multiple interfaces have the same IP address,
a copy of the host endpoint will be resolved with each interface.
"""
# Invert the interface name to IP mapping to allow us to do an IP to
# interface name lookup.
iface_names_by_ip = defaultdict(set)
for iface, ips in self.host_ep_ips_by_iface.iteritems():
for ip in ips:
iface_names_by_ip[ip].add(iface)
# Iterate over the host endpoints, looking for corresponding IPs.
resolved_ifaces = {}
iface_name_to_id = {}
# For repeatability, we sort the endpoint data. We don't care what
# the sort order is, only that it's stable so we just use the repr()
# of the ID.
for combined_id, host_ep in sorted(self.host_eps_by_id.iteritems(),
key=lambda h: repr(h[0])):
addrs_key = "expected_ipv%s_addrs" % self.ip_version
if "name" in host_ep:
# This interface has an explicit name in the data so it's
# already resolved.
resolved_id = combined_id.resolve(host_ep["name"])
resolved_ifaces[resolved_id] = host_ep
elif addrs_key in host_ep:
# No explicit name, look for an interface with a matching IP.
expected_ips = IPSet(host_ep[addrs_key])
for ip, iface_names in sorted(iface_names_by_ip.iteritems()):
if ip in expected_ips:
# This endpoint matches the IP, loop over the (usually
# one) interface with that IP. Sort the names to avoid
# non-deterministic behaviour if there are multiple
# conflicting matches.
_log.debug("Host endpoint %s matches interfaces: %s",
combined_id, iface_names)
for iface_name in sorted(iface_names):
# Check for conflicting matches.
prev_match = iface_name_to_id.get(iface_name)
if prev_match == combined_id:
# Already matched this interface by a different
# IP address.
continue
elif prev_match is not None:
# Already matched a different interface.
# First match wins.
_log.warn("Interface %s matched with multiple "
"entries in datamodel; using %s",
iface_name, prev_match)
continue
else:
# Else, this is the first match, record it.
iface_name_to_id[iface_name] = combined_id
# Got a match. Since it's possible to match
# multiple interfaces by IP, we add the interface
# name into the ID to disambiguate.
resolved_id = combined_id.resolve(iface_name)
resolved_data = host_ep.copy()
resolved_data["name"] = iface_name
resolved_ifaces[resolved_id] = resolved_data
# Fire in deletions for interfaces that no longer resolve.
for resolved_id in self.resolved_host_eps.keys():
if resolved_id not in resolved_ifaces:
_log.debug("%s no longer matches", resolved_id)
self.on_endpoint_update(resolved_id, None)
# Fire in the updates for the new data.
for resolved_id, data in resolved_ifaces.iteritems():
if self.resolved_host_eps.get(resolved_id) != data:
_log.debug("Updating data for %s", resolved_id)
self.on_endpoint_update(resolved_id, data)
# Update the cache so we can calculate deltas next time.
self.resolved_host_eps = resolved_ifaces
def _update_dirty_policy(self):
if not self._data_model_in_sync:
_log.debug("Datamodel not in sync, postponing update to policy")
return
_log.debug("Endpoints with dirty policy: %s",
self.endpoints_with_dirty_policy)
while self.endpoints_with_dirty_policy:
ep_id = self.endpoints_with_dirty_policy.pop()
if self._is_starting_or_live(ep_id):
self._update_tiered_policy(ep_id)
def _update_tiered_policy(self, ep_id):
"""
Sends an updated list of tiered policy to an endpoint.
Recalculates the list.
:param ep_id: ID of the endpoint to send an update to.
"""
_log.debug("Updating policies for %s from %s", ep_id,
self.pol_ids_by_ep_id)
# Order the profiles by tier and profile order, using the name of the
# tier and profile as a tie-breaker if the orders are the same.
profiles = []
for pol_id in self.pol_ids_by_ep_id.iter_values(ep_id):
try:
tier_order = self.tier_orders[pol_id.tier]
except KeyError:
_log.warn("Ignoring profile %s because its tier metadata is "
"missing.")
continue
profile_order = self.profile_orders[pol_id]
profiles.append((tier_order, pol_id.tier,
profile_order, pol_id.policy_id,
pol_id))
profiles.sort()
# Convert to an ordered dict from tier to list of profiles.
pols_by_tier = OrderedDict()
for _, tier, _, _, pol_id in profiles:
pols_by_tier.setdefault(tier, []).append(pol_id)
endpoint = self.objects_by_id[ep_id]
endpoint.on_tiered_policy_update(pols_by_tier, async=True)
def _on_worker_died(self, watch_greenlet):
"""
Greenlet: spawned by the gevent Hub if our worker thread dies.
"""
_log.critical("Worker greenlet died: %s; exiting.", watch_greenlet)
sys.exit(1)
class EndpointManager(ReferenceManager):
def __init__(self, config, ip_type,
iptables_updater,
dispatch_chains,
rules_manager,
fip_manager,
status_reporter):
super(EndpointManager, self).__init__(qualifier=ip_type)
# Configuration and version to use
self.config = config
self.ip_type = ip_type
self.ip_version = futils.IP_TYPE_TO_VERSION[ip_type]
# Peers/utility classes.
self.iptables_updater = iptables_updater
self.dispatch_chains = dispatch_chains
self.rules_mgr = rules_manager
self.status_reporter = status_reporter
self.fip_manager = fip_manager
# All endpoint dicts that are on this host.
self.endpoints_by_id = {}
# Dict that maps from interface name ("tap1234") to endpoint ID.
self.endpoint_id_by_iface_name = {}
# Set of endpoints that are live on this host. I.e. ones that we've
# increffed.
self.local_endpoint_ids = set()
# Index tracking what policy applies to what endpoints.
self.policy_index = LabelValueIndex()
self.policy_index.on_match_started = self.on_policy_match_started
self.policy_index.on_match_stopped = self.on_policy_match_stopped
self._label_inherit_idx = LabelInheritanceIndex(self.policy_index)
# Tier orders by tier ID. We use this to look up the order when we're
# sorting the tiers.
self.tier_orders = {}
# Cache of the current ordering of tier IDs.
self.tier_sequence = []
# And their associated orders.
self.profile_orders = {}
# Set of profile IDs to apply to each endpoint ID.
self.pol_ids_by_ep_id = MultiDict()
self.endpoints_with_dirty_policy = set()
self._data_model_in_sync = False
def _create(self, combined_id):
"""
Overrides ReferenceManager._create()
"""
return LocalEndpoint(self.config,
combined_id,
self.ip_type,
self.iptables_updater,
self.dispatch_chains,
self.rules_mgr,
self.fip_manager,
self.status_reporter)
@actor_message()
def on_tier_data_update(self, tier, data):
"""
Message received when the metadata for a policy tier is updated
in etcd.
:param str tier: The name of the tier.
:param dict|NoneType data: The dict or None, for a deletion.
"""
_log.debug("Data for policy tier %s updated to %s", tier, data)
# Currently, the only data we care about is the order.
order = None if data is None else data["order"]
if self.tier_orders.get(tier) == order:
_log.debug("No change, ignoring")
return
if order is not None:
self.tier_orders[tier] = order
else:
del self.tier_orders[tier]
new_tier_sequence = sorted(self.tier_orders.iterkeys(),
key=lambda k: (self.tier_orders[k], k))
if self.tier_sequence != new_tier_sequence:
_log.info("Sequence of profile tiers changed, refreshing all "
"endpoints")
self.tier_sequence = new_tier_sequence
self.endpoints_with_dirty_policy.update(
self.endpoints_by_id.keys()
)
self._update_dirty_policy()
@actor_message()
def on_prof_labels_set(self, profile_id, labels):
_log.debug("Profile labels updated for %s: %s", profile_id, labels)
# Defer to the label index, which will call us back synchronously
# with any match changes.
self._label_inherit_idx.on_parent_labels_update(profile_id, labels)
# Process any match changes that we've recorded in the callbacks.
self._update_dirty_policy()
@actor_message()
def on_policy_selector_update(self, policy_id, selector_or_none,
order_or_none):
_log.debug("Policy %s selector updated to %s (%s)", policy_id,
selector_or_none, order_or_none)
# Defer to the label index, which will call us back synchronously
# via on_policy_match_started and on_policy_match_stopped.
self.policy_index.on_expression_update(policy_id,
selector_or_none)
# Before we update the policies, check if the order has changed,
# which would mean we need to refresh all endpoints with this policy
# too.
if order_or_none != self.profile_orders.get(policy_id):
if order_or_none is not None:
self.profile_orders[policy_id] = order_or_none
else:
del self.profile_orders[policy_id]
self.endpoints_with_dirty_policy.update(
self.policy_index.matches_by_expr_id.iter_values(policy_id)
)
# Finally, flush any updates to our waiting endpoints.
self._update_dirty_policy()
def on_policy_match_started(self, expr_id, item_id):
"""Called by the label index when a new match is started.
Records the update but processing is deferred to
the next call to self._update_dirty_policy().
"""
_log.info("Policy %s now applies to endpoint %s", expr_id, item_id)
self.pol_ids_by_ep_id.add(item_id, expr_id)
self.endpoints_with_dirty_policy.add(item_id)
def on_policy_match_stopped(self, expr_id, item_id):
"""Called by the label index when a match stops.
Records the update but processing is deferred to
the next call to self._update_dirty_policy().
"""
_log.info("Policy %s no longer applies to endpoint %s",
expr_id, item_id)
self.pol_ids_by_ep_id.discard(item_id, expr_id)
self.endpoints_with_dirty_policy.add(item_id)
def _on_object_started(self, endpoint_id, obj):
"""
Callback from a LocalEndpoint to report that it has started.
Overrides ReferenceManager._on_object_started
"""
ep = self.endpoints_by_id.get(endpoint_id)
obj.on_endpoint_update(ep, async=True)
self._update_tiered_policy(endpoint_id)
@actor_message()
def on_datamodel_in_sync(self):
if not self._data_model_in_sync:
_log.info("%s: First time we've been in-sync with the datamodel,"
"sending snapshot to DispatchChains and FIPManager.",
self)
self._data_model_in_sync = True
# Tell the dispatch chains about the local endpoints in advance so
# that we don't flap the dispatch chain at start-of-day. Note:
# the snapshot may contain information that is ahead of the
# state that our individual LocalEndpoint actors are sending to the
# DispatchChains actor. That is OK! The worst that can happen is
# that a LocalEndpoint undoes part of our update and then goes on
# to re-apply the update when it catches up to the snapshot.
local_ifaces = frozenset(self.endpoint_id_by_iface_name.keys())
self.dispatch_chains.apply_snapshot(local_ifaces, async=True)
self._update_dirty_policy()
nat_maps = {}
for ep_id, ep in self.endpoints_by_id.iteritems():
if ep_id in self.local_endpoint_ids:
nat_map = ep.get(nat_key(self.ip_type), None)
if nat_map:
nat_maps[ep_id] = nat_map
self.fip_manager.apply_snapshot(nat_maps, async=True)
@actor_message()
def on_endpoint_update(self, endpoint_id, endpoint, force_reprogram=False):
"""
Event to indicate that an endpoint has been updated (including
creation or deletion).
:param EndpointId endpoint_id: The endpoint ID in question.
:param dict[str]|NoneType endpoint: Dictionary of all endpoint
data or None if the endpoint is to be deleted.
"""
if endpoint_id.host != self.config.HOSTNAME:
_log.debug("Skipping endpoint %s; not on our host.", endpoint_id)
return
if self._is_starting_or_live(endpoint_id):
# Local endpoint thread is running; tell it of the change.
_log.info("Update for live endpoint %s", endpoint_id)
self.objects_by_id[endpoint_id].on_endpoint_update(
endpoint, force_reprogram=force_reprogram, async=True)
old_ep = self.endpoints_by_id.pop(endpoint_id, {})
# Interface name shouldn't change but popping it now is correct for
# deletes and we add it back in below on create/modify.
old_iface_name = old_ep.get("name")
self.endpoint_id_by_iface_name.pop(old_iface_name, None)
if endpoint is None:
# Deletion. Remove from the list.
_log.info("Endpoint %s deleted", endpoint_id)
if endpoint_id in self.local_endpoint_ids:
self.decref(endpoint_id)
self.local_endpoint_ids.remove(endpoint_id)
self._label_inherit_idx.on_item_update(endpoint_id, None, None)
assert endpoint_id not in self.pol_ids_by_ep_id
else:
# Creation or modification
_log.info("Endpoint %s modified or created", endpoint_id)
self.endpoints_by_id[endpoint_id] = endpoint
self.endpoint_id_by_iface_name[endpoint["name"]] = endpoint_id
if endpoint_id not in self.local_endpoint_ids:
# This will trigger _on_object_activated to pass the endpoint
# we just saved off to the endpoint.
_log.debug("Endpoint wasn't known before, increffing it")
self.local_endpoint_ids.add(endpoint_id)
self.get_and_incref(endpoint_id)
self._label_inherit_idx.on_item_update(
endpoint_id,
endpoint.get("labels", {}),
endpoint.get("profile_ids", [])
)
self._update_dirty_policy()
@actor_message()
def on_interface_update(self, name, iface_up):
"""
Called when an interface is created or changes state.
The interface may be any interface on the host, not necessarily
one managed by any endpoint of this server.
"""
try:
endpoint_id = self.endpoint_id_by_iface_name[name]
except KeyError:
_log.debug("Update on interface %s that we do not care about",
name)
else:
_log.info("Endpoint %s received interface update for %s",
endpoint_id, name)
if self._is_starting_or_live(endpoint_id):
# LocalEndpoint is running, so tell it about the change.
ep = self.objects_by_id[endpoint_id]
ep.on_interface_update(iface_up, async=True)
def _update_dirty_policy(self):
if not self._data_model_in_sync:
_log.debug("Datamodel not in sync, postponing update to policy")
return
_log.debug("Endpoints with dirty policy: %s",
self.endpoints_with_dirty_policy)
while self.endpoints_with_dirty_policy:
ep_id = self.endpoints_with_dirty_policy.pop()
if self._is_starting_or_live(ep_id):
self._update_tiered_policy(ep_id)
def _update_tiered_policy(self, ep_id):
"""
Sends an updated list of tiered policy to an endpoint.
Recalculates the list.
:param ep_id: ID of the endpoint to send an update to.
"""
_log.debug("Updating policies for %s from %s", ep_id,
self.pol_ids_by_ep_id)
# Order the profiles by tier and profile order, using the name of the
# tier and profile as a tie-breaker if the orders are the same.
profiles = []
for pol_id in self.pol_ids_by_ep_id.iter_values(ep_id):
try:
tier_order = self.tier_orders[pol_id.tier]
except KeyError:
_log.warn("Ignoring profile %s because its tier metadata is "
"missing.")
continue
profile_order = self.profile_orders[pol_id]
profiles.append((tier_order, pol_id.tier,
profile_order, pol_id.policy_id,
pol_id))
profiles.sort()
# Convert to an ordered dict from tier to list of profiles.
pols_by_tier = OrderedDict()
for _, tier, _, _, pol_id in profiles:
pols_by_tier.setdefault(tier, []).append(pol_id)
endpoint = self.objects_by_id[ep_id]
endpoint.on_tiered_policy_update(pols_by_tier, async=True)
class EndpointManager(ReferenceManager):
def __init__(self, config, ip_type, iptables_updater, dispatch_chains,
rules_manager, fip_manager, status_reporter):
super(EndpointManager, self).__init__(qualifier=ip_type)
# Configuration and version to use
self.config = config
self.ip_type = ip_type
self.ip_version = futils.IP_TYPE_TO_VERSION[ip_type]
# Peers/utility classes.
self.iptables_updater = iptables_updater
self.dispatch_chains = dispatch_chains
self.rules_mgr = rules_manager
self.status_reporter = status_reporter
self.fip_manager = fip_manager
# All endpoint dicts that are on this host.
self.endpoints_by_id = {}
# Dict that maps from interface name ("tap1234") to endpoint ID.
self.endpoint_id_by_iface_name = {}
# Set of endpoints that are live on this host. I.e. ones that we've
# increffed.
self.local_endpoint_ids = set()
# Index tracking what policy applies to what endpoints.
self.policy_index = LabelValueIndex()
self.policy_index.on_match_started = self.on_policy_match_started
self.policy_index.on_match_stopped = self.on_policy_match_stopped
self._label_inherit_idx = LabelInheritanceIndex(self.policy_index)
# Tier orders by tier ID. We use this to look up the order when we're
# sorting the tiers.
self.tier_orders = {}
# Cache of the current ordering of tier IDs.
self.tier_sequence = []
# And their associated orders.
self.profile_orders = {}
# Set of profile IDs to apply to each endpoint ID.
self.pol_ids_by_ep_id = MultiDict()
self.endpoints_with_dirty_policy = set()
self._data_model_in_sync = False
def _create(self, combined_id):
"""
Overrides ReferenceManager._create()
"""
return LocalEndpoint(self.config, combined_id, self.ip_type,
self.iptables_updater, self.dispatch_chains,
self.rules_mgr, self.fip_manager,
self.status_reporter)
@actor_message()
def on_tier_data_update(self, tier, data):
"""
Message received when the metadata for a policy tier is updated
in etcd.
:param str tier: The name of the tier.
:param dict|NoneType data: The dict or None, for a deletion.
"""
_log.debug("Data for policy tier %s updated to %s", tier, data)
# Currently, the only data we care about is the order.
order = None if data is None else data["order"]
if self.tier_orders.get(tier) == order:
_log.debug("No change, ignoring")
return
if order is not None:
self.tier_orders[tier] = order
else:
del self.tier_orders[tier]
new_tier_sequence = sorted(self.tier_orders.iterkeys(),
key=lambda k: (self.tier_orders[k], k))
if self.tier_sequence != new_tier_sequence:
_log.info("Sequence of profile tiers changed, refreshing all "
"endpoints")
self.tier_sequence = new_tier_sequence
self.endpoints_with_dirty_policy.update(
self.endpoints_by_id.keys())
self._update_dirty_policy()
@actor_message()
def on_prof_labels_set(self, profile_id, labels):
_log.debug("Profile labels updated for %s: %s", profile_id, labels)
# Defer to the label index, which will call us back synchronously
# with any match changes.
self._label_inherit_idx.on_parent_labels_update(profile_id, labels)
# Process any match changes that we've recorded in the callbacks.
self._update_dirty_policy()
@actor_message()
def on_policy_selector_update(self, policy_id, selector_or_none,
order_or_none):
_log.debug("Policy %s selector updated to %s (%s)", policy_id,
selector_or_none, order_or_none)
# Defer to the label index, which will call us back synchronously
# via on_policy_match_started and on_policy_match_stopped.
self.policy_index.on_expression_update(policy_id, selector_or_none)
# Before we update the policies, check if the order has changed,
# which would mean we need to refresh all endpoints with this policy
# too.
if order_or_none != self.profile_orders.get(policy_id):
if order_or_none is not None:
self.profile_orders[policy_id] = order_or_none
else:
del self.profile_orders[policy_id]
self.endpoints_with_dirty_policy.update(
self.policy_index.matches_by_expr_id.iter_values(policy_id))
# Finally, flush any updates to our waiting endpoints.
self._update_dirty_policy()
def on_policy_match_started(self, expr_id, item_id):
"""Called by the label index when a new match is started.
Records the update but processing is deferred to
the next call to self._update_dirty_policy().
"""
_log.info("Policy %s now applies to endpoint %s", expr_id, item_id)
self.pol_ids_by_ep_id.add(item_id, expr_id)
self.endpoints_with_dirty_policy.add(item_id)
def on_policy_match_stopped(self, expr_id, item_id):
"""Called by the label index when a match stops.
Records the update but processing is deferred to
the next call to self._update_dirty_policy().
"""
_log.info("Policy %s no longer applies to endpoint %s", expr_id,
item_id)
self.pol_ids_by_ep_id.discard(item_id, expr_id)
self.endpoints_with_dirty_policy.add(item_id)
def _on_object_started(self, endpoint_id, obj):
"""
Callback from a LocalEndpoint to report that it has started.
Overrides ReferenceManager._on_object_started
"""
ep = self.endpoints_by_id.get(endpoint_id)
obj.on_endpoint_update(ep, async=True)
self._update_tiered_policy(endpoint_id)
@actor_message()
def on_datamodel_in_sync(self):
if not self._data_model_in_sync:
_log.info(
"%s: First time we've been in-sync with the datamodel,"
"sending snapshot to DispatchChains and FIPManager.", self)
self._data_model_in_sync = True
# Tell the dispatch chains about the local endpoints in advance so
# that we don't flap the dispatch chain at start-of-day. Note:
# the snapshot may contain information that is ahead of the
# state that our individual LocalEndpoint actors are sending to the
# DispatchChains actor. That is OK! The worst that can happen is
# that a LocalEndpoint undoes part of our update and then goes on
# to re-apply the update when it catches up to the snapshot.
local_ifaces = frozenset(self.endpoint_id_by_iface_name.keys())
self.dispatch_chains.apply_snapshot(local_ifaces, async=True)
self._update_dirty_policy()
nat_maps = {}
for ep_id, ep in self.endpoints_by_id.iteritems():
if ep_id in self.local_endpoint_ids:
nat_map = ep.get(nat_key(self.ip_type), None)
if nat_map:
nat_maps[ep_id] = nat_map
self.fip_manager.apply_snapshot(nat_maps, async=True)
@actor_message()
def on_endpoint_update(self, endpoint_id, endpoint, force_reprogram=False):
"""
Event to indicate that an endpoint has been updated (including
creation or deletion).
:param EndpointId endpoint_id: The endpoint ID in question.
:param dict[str]|NoneType endpoint: Dictionary of all endpoint
data or None if the endpoint is to be deleted.
"""
if endpoint_id.host != self.config.HOSTNAME:
_log.debug("Skipping endpoint %s; not on our host.", endpoint_id)
return
if self._is_starting_or_live(endpoint_id):
# Local endpoint thread is running; tell it of the change.
_log.info("Update for live endpoint %s", endpoint_id)
self.objects_by_id[endpoint_id].on_endpoint_update(
endpoint, force_reprogram=force_reprogram, async=True)
old_ep = self.endpoints_by_id.pop(endpoint_id, {})
# Interface name shouldn't change but popping it now is correct for
# deletes and we add it back in below on create/modify.
old_iface_name = old_ep.get("name")
self.endpoint_id_by_iface_name.pop(old_iface_name, None)
if endpoint is None:
# Deletion. Remove from the list.
_log.info("Endpoint %s deleted", endpoint_id)
if endpoint_id in self.local_endpoint_ids:
self.decref(endpoint_id)
self.local_endpoint_ids.remove(endpoint_id)
self._label_inherit_idx.on_item_update(endpoint_id, None, None)
assert endpoint_id not in self.pol_ids_by_ep_id
else:
# Creation or modification
_log.info("Endpoint %s modified or created", endpoint_id)
self.endpoints_by_id[endpoint_id] = endpoint
self.endpoint_id_by_iface_name[endpoint["name"]] = endpoint_id
if endpoint_id not in self.local_endpoint_ids:
# This will trigger _on_object_activated to pass the endpoint
# we just saved off to the endpoint.
_log.debug("Endpoint wasn't known before, increffing it")
self.local_endpoint_ids.add(endpoint_id)
self.get_and_incref(endpoint_id)
self._label_inherit_idx.on_item_update(
endpoint_id, endpoint.get("labels", {}),
endpoint.get("profile_ids", []))
self._update_dirty_policy()
@actor_message()
def on_interface_update(self, name, iface_up):
"""
Called when an interface is created or changes state.
The interface may be any interface on the host, not necessarily
one managed by any endpoint of this server.
"""
try:
endpoint_id = self.endpoint_id_by_iface_name[name]
except KeyError:
_log.debug("Update on interface %s that we do not care about",
name)
else:
_log.info("Endpoint %s received interface update for %s",
endpoint_id, name)
if self._is_starting_or_live(endpoint_id):
# LocalEndpoint is running, so tell it about the change.
ep = self.objects_by_id[endpoint_id]
ep.on_interface_update(iface_up, async=True)
def _update_dirty_policy(self):
if not self._data_model_in_sync:
_log.debug("Datamodel not in sync, postponing update to policy")
return
_log.debug("Endpoints with dirty policy: %s",
self.endpoints_with_dirty_policy)
while self.endpoints_with_dirty_policy:
ep_id = self.endpoints_with_dirty_policy.pop()
if self._is_starting_or_live(ep_id):
self._update_tiered_policy(ep_id)
def _update_tiered_policy(self, ep_id):
"""
Sends an updated list of tiered policy to an endpoint.
Recalculates the list.
:param ep_id: ID of the endpoint to send an update to.
"""
_log.debug("Updating policies for %s from %s", ep_id,
self.pol_ids_by_ep_id)
# Order the profiles by tier and profile order, using the name of the
# tier and profile as a tie-breaker if the orders are the same.
profiles = []
for pol_id in self.pol_ids_by_ep_id.iter_values(ep_id):
try:
tier_order = self.tier_orders[pol_id.tier]
except KeyError:
_log.warn("Ignoring profile %s because its tier metadata is "
"missing.")
continue
profile_order = self.profile_orders[pol_id]
profiles.append((tier_order, pol_id.tier, profile_order,
pol_id.policy_id, pol_id))
profiles.sort()
# Convert to an ordered dict from tier to list of profiles.
pols_by_tier = OrderedDict()
for _, tier, _, _, pol_id in profiles:
pols_by_tier.setdefault(tier, []).append(pol_id)
endpoint = self.objects_by_id[ep_id]
endpoint.on_tiered_policy_update(pols_by_tier, async=True)
class TestMultiDict(TestCase):
def setUp(self):
super(TestMultiDict, self).setUp()
self.index = MultiDict()
def test_add_single(self):
self.index.add("k", "v")
self.assertTrue(self.index.contains("k", "v"))
self.assertEqual(set(self.index.iter_values("k")), set(["v"]))
def test_add_remove_single(self):
self.index.add("k", "v")
self.index.discard("k", "v")
self.assertFalse(self.index.contains("k", "v"))
self.assertEqual(self.index._index, {})
def test_empty(self):
self.assertFalse(bool(self.index))
self.assertEqual(self.index.num_items("k"), 0)
self.assertEqual(list(self.index.iter_values("k")), [])
def test_add_multiple(self):
self.index.add("k", "v")
self.assertTrue(bool(self.index))
self.assertEqual(self.index.num_items("k"), 1)
self.index.add("k", "v")
self.assertEqual(self.index.num_items("k"), 1)
self.index.add("k", "v2")
self.assertEqual(self.index.num_items("k"), 2)
self.index.add("k", "v3")
self.assertEqual(self.index.num_items("k"), 3)
self.assertIn("k", self.index)
self.assertNotIn("k2", self.index)
self.assertTrue(self.index.contains("k", "v"))
self.assertTrue(self.index.contains("k", "v2"))
self.assertTrue(self.index.contains("k", "v3"))
self.assertEqual(self.index._index, {"k": set(["v", "v2", "v3"])})
self.assertEqual(set(self.index.iter_values("k")),
set(["v", "v2", "v3"]))
self.index.discard("k", "v")
self.index.discard("k", "v2")
self.assertTrue(self.index.contains("k", "v3"))
self.index.discard("k", "v3")
self.assertEqual(self.index._index, {})
class TestMultiDict(TestCase):
def setUp(self):
super(TestMultiDict, self).setUp()
self.index = MultiDict()
def test_add_single(self):
self.index.add("k", "v")
self.assertTrue(self.index.contains("k", "v"))
self.assertEqual(set(self.index.iter_values("k")),
set(["v"]))
def test_add_remove_single(self):
self.index.add("k", "v")
self.index.discard("k", "v")
self.assertFalse(self.index.contains("k", "v"))
self.assertEqual(self.index._index, {})
def test_empty(self):
self.assertFalse(bool(self.index))
self.assertEqual(self.index.num_items("k"), 0)
self.assertEqual(list(self.index.iter_values("k")), [])
def test_add_multiple(self):
self.index.add("k", "v")
self.assertTrue(bool(self.index))
self.assertEqual(self.index.num_items("k"), 1)
self.index.add("k", "v")
self.assertEqual(self.index.num_items("k"), 1)
self.index.add("k", "v2")
self.assertEqual(self.index.num_items("k"), 2)
self.index.add("k", "v3")
self.assertEqual(self.index.num_items("k"), 3)
self.assertIn("k", self.index)
self.assertNotIn("k2", self.index)
self.assertTrue(self.index.contains("k", "v"))
self.assertTrue(self.index.contains("k", "v2"))
self.assertTrue(self.index.contains("k", "v3"))
self.assertEqual(self.index._index, {"k": set(["v", "v2", "v3"])})
self.assertEqual(set(self.index.iter_values("k")),
set(["v", "v2", "v3"]))
self.index.discard("k", "v")
self.index.discard("k", "v2")
self.assertTrue(self.index.contains("k", "v3"))
self.index.discard("k", "v3")
self.assertEqual(self.index._index, {})