def test_helper_for_securitygroup_informer(self):
'''Test a CidrCatalogHelper instance with a Security Group Informer.'''
informers = self.sg_surv.informers()
# - - - - - - - - - - - - - - - -
informers_with_cidrip = [
i for i in informers
if prune.Pruner().leaf_satisfies(i.to_dict(
), 'IpPermissions.[].IpRanges.[]', lambda x: 'CidrIp' in x)
]
self.assertTrue(len(informers_with_cidrip) > 0)
self.helper.annotate_group_cidrip(informers_with_cidrip[0], 'name')
self.assertTrue(prune.Pruner().leaf_satisfies(
informers_with_cidrip[0].to_dict(), 'IpPermissions.[].IpRanges.[]',
lambda x: 'name' in x))
self.helper.annotate_group_cidrip(informers_with_cidrip[0],
'short_name', 'description')
self.assertTrue(prune.Pruner().leaf_satisfies(
informers_with_cidrip[0].to_dict(), 'IpPermissions.[].IpRanges.[]',
lambda x: 'short_name' in x and 'description' in x))
# - - - - - - - - - - - - - - - -
informers_with_egress_cidrip = [
i for i in informers
if prune.Pruner().leaf_satisfies(i.to_dict(
), 'IpPermissions.[].IpRanges.[]', lambda x: 'CidrIp' in x)
]
self.assertTrue(len(informers_with_egress_cidrip) > 0)
self.helper.annotate_group_cidrip(informers_with_egress_cidrip[0],
'name')
self.assertTrue(prune.Pruner().leaf_satisfies(
informers_with_egress_cidrip[0].to_dict(),
'IpPermissions.[].IpRanges.[]', lambda x: 'name' in x))
def extract_from(self, informers, flat=True):
'''Extract the fields specified by prune_specs.
Arguments:
informers (list of AWSInformer):
``AWSInformer`` instances from which to extract
report data. Any informer whose ``entity_type``
attribute doesn't match the report definition's
``entity_type`` attribute will be skipped.
flat (bool):
If ``True``, return a *flat* result; if ``False``,
return a nested result. See the documentation for
``utensils.prune`` and ``utensils.flatten``
for more information.
'''
pruner = prune.Pruner(*self.prune_specs)
extractable_informers = [
i for i in informers if i.entity_type == self.entity_type
]
if flat:
records = flatten.flatten([
pruner.prune_branches(informer.to_dict(), balanced=True)
for informer in extractable_informers
])
else:
records = [
pruner.prune_tree(informer.to_dict())
for informer in extractable_informers
]
return records
def uid_index(forest, uid_path=None, uid_method=None):
'''Index a set of nested structures by a specified unique id.
Arguments:
forest (list):
A list of nested structures composed of lists, dicts and
scalars to be indexed by unique id.
uid_path (str):
A prune path specification specifying the path to the
unique ID value for each structure in ``forest``. See the
``utensils.prune`` module documentation for details.
uid_method (function):
A function that takes a single argument and when passed a
structure in ``forest`` returns the unique ID value for
that structure.
Exactly one of uid_path and uid_method is required.
Returns:
(dict) A dictionary where each item key is the value selected
by ``uid_path`` in each element of ``forest``, and the
corresponding item value is the element of ``forest`` from
which that value was selected.
Raises:
KeyError: if the value selected by ``uid_path`` from an
element of ``forest`` was already selected from a previous
element of ``forest``. This would indicate a duplication of
the supposedly unique IDs.
TypeError: if neither uid_path nor uid_method is provided, or
if both are.
'''
logger = logging.getLogger(__name__)
index = {}
# Exactly one of uid_path and uid_method must be defined.
defined_extractor_count = 2 - (uid_path, uid_method).count(None)
if defined_extractor_count != 1:
err_msg = (
'uid_index: exactly one of uid_path and uid_method is required;'
' found %s' % ('neither', '', 'both')[defined_extractor_count]
)
logger.error(err_msg)
raise TypeError(err_msg)
for tree in forest:
index_keys = []
if uid_path:
# It's unclear whether this ought to enforce getting a
# single element list for index_keys.
pruner = prune.Pruner({'path': prune.dotpath(uid_path)})
for (_, pruned_keys) in pruner.prune_leaves(tree).items():
index_keys.extend(pruned_keys)
else:
# Wrap this for consistency with the prune_leaves return
# value type.
index_keys = [uid_method(tree)]
for index_key in list(set(index_keys)):
if index_key in index:
msg = (
'uid_index: duplicate occurrence of unique ID %s'
)
logger.error(msg, index_key)
raise KeyError(msg % index_key)
index[index_key] = tree
return index
def index_subtrees_by_paths(
forest, paths, subtree_paths, duplicates=False,
prep_method=None, key_method=None, value_method=None,
): # pylint: disable=bad-continuation
'''Build indices of subtrees of nested structures.
Arguments:
forest:
A nested structure composed of lists, dicts and scalars.
paths:
A list of prune path specifications. See the
``utensils.prune`` module documentation for details.
subtree_paths:
A list of prune-style path specifications to the subelements
to be indexed.
duplicates (boolean, optional):
If ``False`` (the default), no duplicate values will
appear in the lists of values for each key in the indices
returned. If ``True``, duplicates can occur.
prep_method (function):
If defined, each element of ``forest`` will be replaced
locally with the result of passing it to the
``prep_method`` method. Thus,
``index_subtrees_by_paths([t1, t2], p, sp, prep_method=m)``
will return the same value as
``index_subtrees_by_paths([m(t1), m(t2)], p, sp)``.
key_method (function, optional):
If defined, each key ``K`` that would appear as a key in
the resulting index will be replaced by the result of
evaluating ``key_method(K)``.
value_method (function, optional):
If defined, each value ``V`` that would appear in the
list of values for any key in the resulting index will be
replaced by the result of evaluating ``value_method(V)``.
Returns:
(dict) A dict whose keys are the list-reduced elements of paths,
and whose values are dicts. The structure will look like this::
{
index_path:
{
path_valueA: [subtree1, subtree2, ...],
path_valueB: [...],
...
},
...
}
Here, the path_valueX keys are the values that occur at path
in any of the elements of forest, and the list for each key is
the specified subtrees taken from the elements of forest for
which the path_valueX value occurs at path.
'''
index_pruner = prune.Pruner(
*[{'path': prune.dotpath(p)} for p in paths]
)
subtree_pruner = prune.Pruner(
*[{'path': prune.dotpath(p)} for p in subtree_paths]
)
indices = {}
# print [{'path': p} for p in paths]
for tree in forest:
prepped_tree = prep_method(tree) if prep_method else tree
index_leaves = index_pruner.prune_leaves(prepped_tree)
subtrees = subtree_pruner.prune_leaves(prepped_tree)
# print subtrees
for (index_path, path_values) in index_leaves.items():
if index_path not in indices:
indices[index_path] = {}
# for path_value in list(set(path_values)):
for index_key in list(set(
[key_method(v) if key_method else v for v in path_values]
)): # pylint: disable=bad-continuation
if index_key not in indices[index_path]:
indices[index_path][index_key] = []
for (_, subtree) in subtrees.items():
if value_method:
subtree = [value_method(t) for t in subtree]
# if (
# value not in indices[index_path][index_key] or
# duplicates
# ): # pylint: disable=bad-continuation
# # print value
# # print indices[index_path][index_key]
# # print value not in indices[index_path][index_key]
indices[index_path][index_key].extend(
[
v for v in subtree
if (
v not in indices[index_path][index_key] or
duplicates
)
]
)
# print indices[index_path][index_key]
# print
return indices
def index_by_paths(
forest, paths, duplicates=False,
prep_method=None, key_method=None, value_method=None,
): # pylint: disable=bad-continuation
'''Build indices by values at specified paths into nested structures.
Arguments:
forest (list):
A list of nested structures composed of lists, dicts and
scalars.
paths (list):
A list of prune path specifications. See the
``utensils.prune`` module documentation for details.
duplicates (boolean, optional):
If ``False`` (the default), no duplicate values will
appear in the lists of values for each key in the indices
returned. If ``True``, duplicates can occur.
prep_method (function, optional):
If defined, the indices' keys will be the values at the
specified paths in the output of ``prep_method`` applied
to each member of ``forest``. The values of the indices
will be the original members of ``forest``.
key_method (function, optional):
If defined, each key ``K` that would appear as a key in
the resulting indices will be replaced by the result of
evaluating ``key_method(K)``.
value_method (function, optional):
If defined, each value ``V`` that would appear in the
list of values for any key in the resulting indices will
be replaced by the result of evaluating
``value_method(V)``.
Returns:
(dict) A dictionary in which each value is an index of the
structures in ``forest`` by the values of the elements
selected by one ``path`` in ``paths``. The dictionary key for
a given index value will be the "list-reduced dot separated"
representation of ``path``. This is the string returned by
the function call
``utensils.prune.dotpath(path, no_lists=True)``.
The resulting structure will look like this::
{
path:
{
path_valueA: [tree1, tree2, ...],
path_valueB: [...],
...
},
...
}
Here, the path_valueX keys are the values that occur at path
in any of the elements of forest, and the list for each key is
exactly the elements of forest for which the path_valueX value
occurs at path. Note that the values selected by a path in
``paths``, or the values returned by ``key_method()``, if defined,
must be hashable, as they will be keys in an index.
'''
pruner = prune.Pruner(
*[{'path': prune.dotpath(p)} for p in paths]
)
indices = {}
# print [{'path': p} for p in paths]
for tree in forest:
prepped_tree = prep_method(tree) if prep_method else tree
for (path, path_values) in pruner.prune_leaves(prepped_tree).items():
if path not in indices:
indices[path] = {}
for index_key in list(set(
[key_method(v) if key_method else v for v in path_values]
)): # pylint: disable=bad-continuation
if index_key not in indices[path]:
indices[path][index_key] = []
value = value_method(tree) if value_method else tree
if value not in indices[path][index_key] or duplicates:
indices[path][index_key].append(value)
return indices