def subelements(obj, subelements, skip_missing=False):
'''Accepts a dict or list of dicts, and a dotted accessor and produces a product
of the element and the results of the dotted accessor
>>> obj = [{"name": "alice", "groups": ["wheel"], "authorized": ["/tmp/alice/onekey.pub"]}]
>>> subelements(obj, 'groups')
[({'name': 'alice', 'groups': ['wheel'], 'authorized': ['/tmp/alice/onekey.pub']}, 'wheel')]
'''
if isinstance(obj, dict):
element_list = list(obj.values())
elif isinstance(obj, list):
element_list = obj[:]
else:
raise AssibleFilterError(
'obj must be a list of dicts or a nested dict')
if isinstance(subelements, list):
subelement_list = subelements[:]
elif isinstance(subelements, string_types):
subelement_list = subelements.split('.')
else:
raise AssibleFilterTypeError('subelements must be a list or a string')
results = []
for element in element_list:
values = element
for subelement in subelement_list:
try:
values = values[subelement]
except KeyError:
if skip_missing:
values = []
break
raise AssibleFilterError(
"could not find %r key in iterated item %r" %
(subelement, values))
except TypeError:
raise AssibleFilterTypeError(
"the key %s should point to a dictionary, got '%s'" %
(subelement, values))
if not isinstance(values, list):
raise AssibleFilterTypeError(
"the key %r should point to a list, got %r" %
(subelement, values))
for value in values:
results.append((element, value))
return results
def rekey_on_member(data, key, duplicates='error'):
"""
Rekey a dict of dicts on another member
May also create a dict from a list of dicts.
duplicates can be one of ``error`` or ``overwrite`` to specify whether to error out if the key
value would be duplicated or to overwrite previous entries if that's the case.
"""
if duplicates not in ('error', 'overwrite'):
raise AssibleFilterError(
"duplicates parameter to rekey_on_member has unknown value: {0}".
format(duplicates))
new_obj = {}
if isinstance(data, Mapping):
iterate_over = data.values()
elif isinstance(
data, Iterable) and not isinstance(data, (text_type, binary_type)):
iterate_over = data
else:
raise AssibleFilterTypeError("Type is not a valid list, set, or dict")
for item in iterate_over:
if not isinstance(item, Mapping):
raise AssibleFilterTypeError("List item is not a valid dict")
try:
key_elem = item[key]
except KeyError:
raise AssibleFilterError("Key {0} was not found".format(key))
except TypeError as e:
raise AssibleFilterTypeError(to_native(e))
except Exception as e:
raise AssibleFilterError(to_native(e))
# Note: if new_obj[key_elem] exists it will always be a non-empty dict (it will at
# minimum contain {key: key_elem}
if new_obj.get(key_elem, None):
if duplicates == 'error':
raise AssibleFilterError(
"Key {0} is not unique, cannot correctly turn into dict".
format(key_elem))
elif duplicates == 'overwrite':
new_obj[key_elem] = item
else:
new_obj[key_elem] = item
return new_obj
def combine(*terms, **kwargs):
recursive = kwargs.pop('recursive', False)
list_merge = kwargs.pop('list_merge', 'replace')
if kwargs:
raise AssibleFilterError(
"'recursive' and 'list_merge' are the only valid keyword arguments"
)
# allow the user to do `[dict1, dict2, ...] | combine`
dictionaries = flatten(terms, levels=1)
# recursively check that every elements are defined (for jinja2)
recursive_check_defined(dictionaries)
if not dictionaries:
return {}
if len(dictionaries) == 1:
return dictionaries[0]
# merge all the dicts so that the dict at the end of the array have precedence
# over the dict at the beginning.
# we merge the dicts from the highest to the lowest priority because there is
# a huge probability that the lowest priority dict will be the biggest in size
# (as the low prio dict will hold the "default" values and the others will be "patches")
# and merge_hash create a copy of it's first argument.
# so high/right -> low/left is more efficient than low/left -> high/right
high_to_low_prio_dict_iterator = reversed(dictionaries)
result = next(high_to_low_prio_dict_iterator)
for dictionary in high_to_low_prio_dict_iterator:
result = merge_hash(dictionary, result, recursive, list_merge)
return result
def regex_search(value, regex, *args, **kwargs):
''' Perform re.search and return the list of matches or a backref '''
value = to_text(value,
errors='surrogate_or_strict',
nonstring='simplerepr')
groups = list()
for arg in args:
if arg.startswith('\\g'):
match = re.match(r'\\g<(\S+)>', arg).group(1)
groups.append(match)
elif arg.startswith('\\'):
match = int(re.match(r'\\(\d+)', arg).group(1))
groups.append(match)
else:
raise AssibleFilterError('Unknown argument')
flags = 0
if kwargs.get('ignorecase'):
flags |= re.I
if kwargs.get('multiline'):
flags |= re.M
match = re.search(regex, value, flags)
if match:
if not groups:
return match.group()
else:
items = list()
for item in groups:
items.append(match.group(item))
return items
def mandatory(a, msg=None):
from jinja2.runtime import Undefined
''' Make a variable mandatory '''
if isinstance(a, Undefined):
if a._undefined_name is not None:
name = "'%s' " % to_text(a._undefined_name)
else:
name = ''
if msg is not None:
raise AssibleFilterError(to_native(msg))
else:
raise AssibleFilterError("Mandatory variable %s not defined." %
name)
return a
def json_query(data, expr):
'''Query data using jmespath query language ( http://jmespath.org ). Example:
- debug: msg="{{ instance | json_query(tagged_instances[*].block_device_mapping.*.volume_id') }}"
'''
if not HAS_LIB:
raise AssibleError('You need to install "jmespath" prior to running '
'json_query filter')
try:
return jmespath.search(expr, data)
except jmespath.exceptions.JMESPathError as e:
raise AssibleFilterError(
'JMESPathError in json_query filter plugin:\n%s' % e)
except Exception as e:
# For older jmespath, we can get ValueError and TypeError without much info.
raise AssibleFilterError(
'Error in jmespath.search in json_query filter plugin:\n%s' % e)
def strftime(string_format, second=None):
''' return a date string using string. See https://docs.python.org/2/library/time.html#time.strftime for format '''
if second is not None:
try:
second = float(second)
except Exception:
raise AssibleFilterError('Invalid value for epoch value (%s)' %
second)
return time.strftime(string_format, time.localtime(second))
def hash_salt(password):
split_password = password.split("$")
if len(split_password) != 4:
raise AssibleFilterError(
"Could not parse salt out password correctly from {0}".format(
password))
else:
return split_password[2]
def get_hash(data, hashtype='sha1'):
try:
h = hashlib.new(hashtype)
except Exception as e:
# hash is not supported?
raise AssibleFilterError(e)
h.update(to_bytes(data, errors='surrogate_or_strict'))
return h.hexdigest()
def human_readable(size, isbits=False, unit=None):
''' Return a human readable string '''
try:
return formatters.bytes_to_human(size, isbits, unit)
except TypeError as e:
raise AssibleFilterTypeError(
"human_readable() failed on bad input: %s" % to_native(e))
except Exception:
raise AssibleFilterError(
"human_readable() can't interpret following string: %s" % size)
def rand(environment, end, start=None, step=None, seed=None):
if seed is None:
r = SystemRandom()
else:
r = Random(seed)
if isinstance(end, integer_types):
if not start:
start = 0
if not step:
step = 1
return r.randrange(start, end, step)
elif hasattr(end, '__iter__'):
if start or step:
raise AssibleFilterError(
'start and step can only be used with integer values')
return r.choice(end)
else:
raise AssibleFilterError(
'random can only be used on sequences and integers')
def human_to_bytes(size, default_unit=None, isbits=False):
''' Return bytes count from a human readable string '''
try:
return formatters.human_to_bytes(size, default_unit, isbits)
except TypeError as e:
raise AssibleFilterTypeError(
"human_to_bytes() failed on bad input: %s" % to_native(e))
except Exception:
raise AssibleFilterError(
"human_to_bytes() can't interpret following string: %s" % size)
def regex_escape(string, re_type='python'):
string = to_text(string,
errors='surrogate_or_strict',
nonstring='simplerepr')
'''Escape all regular expressions special characters from STRING.'''
if re_type == 'python':
return re.escape(string)
elif re_type == 'posix_basic':
# list of BRE special chars:
# https://en.wikibooks.org/wiki/Regular_Expressions/POSIX_Basic_Regular_Expressions
return regex_replace(string, r'([].[^$*\\])', r'\\\1')
# TODO: implement posix_extended
# It's similar to, but different from python regex, which is similar to,
# but different from PCRE. It's possible that re.escape would work here.
# https://remram44.github.io/regex-cheatsheet/regex.html#programs
elif re_type == 'posix_extended':
raise AssibleFilterError('Regex type (%s) not yet implemented' %
re_type)
else:
raise AssibleFilterError('Invalid regex type (%s)' % re_type)
def max(environment, a, **kwargs):
if HAS_MIN_MAX:
return do_max(environment, a, **kwargs)
else:
if kwargs:
raise AssibleFilterError(
"Assible's max filter does not support any keyword arguments. "
"You need Jinja2 2.10 or later that provides their version of the filter."
)
_max = __builtins__.get('max')
return _max(a)
def recursive_check_defined(item):
from jinja2.runtime import Undefined
if isinstance(item, MutableMapping):
for key in item:
recursive_check_defined(item[key])
elif isinstance(item, list):
for i in item:
recursive_check_defined(i)
else:
if isinstance(item, Undefined):
raise AssibleFilterError("{0} is undefined".format(item))
def to_uuid(string, namespace=UUID_NAMESPACE_ASSIBLE):
uuid_namespace = namespace
if not isinstance(uuid_namespace, uuid.UUID):
try:
uuid_namespace = uuid.UUID(namespace)
except (AttributeError, ValueError) as e:
raise AssibleFilterError("Invalid value '%s' for 'namespace': %s" %
(to_native(namespace), to_native(e)))
# uuid.uuid5() requires bytes on Python 2 and bytes or text or Python 3
return to_text(
uuid.uuid5(uuid_namespace,
to_native(string, errors='surrogate_or_strict')))
def type5_pw(password, salt=None):
if not isinstance(password, string_types):
raise AssibleFilterError(
"type5_pw password input should be a string, but was given a input of %s"
% (type(password).__name__))
salt_chars = u"".join(
(to_text(string.ascii_letters), to_text(string.digits), u"./"))
if salt is not None and not isinstance(salt, string_types):
raise AssibleFilterError(
"type5_pw salt input should be a string, but was given a input of %s"
% (type(salt).__name__))
elif not salt:
salt = random_password(length=4, chars=salt_chars)
elif not set(salt) <= set(salt_chars):
raise AssibleFilterError(
"type5_pw salt used inproper characters, must be one of %s" %
(salt_chars))
encrypted_password = passlib_or_crypt(password, "md5_crypt", salt=salt)
return encrypted_password
def split_url(value, query='', alias='urlsplit'):
results = helpers.object_to_dict(
urlsplit(value), exclude=['count', 'index', 'geturl', 'encode'])
# If a query is supplied, make sure it's valid then return the results.
# If no option is supplied, return the entire dictionary.
if query:
if query not in results:
raise AssibleFilterError(alias +
': unknown URL component: %s' % query)
return results[query]
else:
return results
def unique(environment, a, case_sensitive=False, attribute=None):
def _do_fail(e):
if case_sensitive or attribute:
raise AssibleFilterError(
"Jinja2's unique filter failed and we cannot fall back to Assible's version "
"as it does not support the parameters supplied",
orig_exc=e)
error = e = None
try:
if HAS_UNIQUE:
c = do_unique(environment,
a,
case_sensitive=case_sensitive,
attribute=attribute)
if isinstance(a, Hashable):
c = set(c)
else:
c = list(c)
except TypeError as e:
error = e
_do_fail(e)
except Exception as e:
error = e
_do_fail(e)
display.warning(
'Falling back to Assible unique filter as Jinja2 one failed: %s' %
to_text(e))
if not HAS_UNIQUE or error:
# handle Jinja2 specific attributes when using Assible's version
if case_sensitive or attribute:
raise AssibleFilterError(
"Assible's unique filter does not support case_sensitive nor attribute parameters, "
"you need a newer version of Jinja2 that provides their version of the filter."
)
if isinstance(a, Hashable):
c = set(a)
else:
c = []
for x in a:
if x not in c:
c.append(x)
return c
def get_encrypted_password(password,
hashtype='sha512',
salt=None,
salt_size=None,
rounds=None):
passlib_mapping = {
'md5': 'md5_crypt',
'blowfish': 'bcrypt',
'sha256': 'sha256_crypt',
'sha512': 'sha512_crypt',
}
hashtype = passlib_mapping.get(hashtype, hashtype)
try:
return passlib_or_crypt(password,
hashtype,
salt=salt,
salt_size=salt_size,
rounds=rounds)
except AssibleError as e:
reraise(AssibleFilterError, AssibleFilterError(to_native(e),
orig_exc=e),
sys.exc_info()[2])
def vlan_parser(vlan_list, first_line_len=48, other_line_len=44):
"""
Input: Unsorted list of vlan integers
Output: Sorted string list of integers according to IOS-like vlan list rules
1. Vlans are listed in ascending order
2. Runs of 3 or more consecutive vlans are listed with a dash
3. The first line of the list can be first_line_len characters long
4. Subsequent list lines can be other_line_len characters
"""
# Sort and remove duplicates
sorted_list = sorted(set(vlan_list))
if sorted_list[0] < 1 or sorted_list[-1] > 4094:
raise AssibleFilterError("Valid VLAN range is 1-4094")
parse_list = []
idx = 0
while idx < len(sorted_list):
start = idx
end = start
while end < len(sorted_list) - 1:
if sorted_list[end + 1] - sorted_list[end] == 1:
end += 1
else:
break
if start == end:
# Single VLAN
parse_list.append(str(sorted_list[idx]))
elif start + 1 == end:
# Run of 2 VLANs
parse_list.append(str(sorted_list[start]))
parse_list.append(str(sorted_list[end]))
else:
# Run of 3 or more VLANs
parse_list.append(
str(sorted_list[start]) + "-" + str(sorted_list[end]))
idx = end + 1
line_count = 0
result = [""]
for vlans in parse_list:
# First line (" switchport trunk allowed vlan ")
if line_count == 0:
if len(result[line_count] + vlans) > first_line_len:
result.append("")
line_count += 1
result[line_count] += vlans + ","
else:
result[line_count] += vlans + ","
# Subsequent lines (" switchport trunk allowed vlan add ")
else:
if len(result[line_count] + vlans) > other_line_len:
result.append("")
line_count += 1
result[line_count] += vlans + ","
else:
result[line_count] += vlans + ","
# Remove trailing orphan commas
for idx in range(0, len(result)):
result[idx] = result[idx].rstrip(",")
# Sometimes text wraps to next line, but there are no remaining VLANs
if "" in result:
result.remove("")
return result
def _do_fail(e):
if case_sensitive or attribute:
raise AssibleFilterError(
"Jinja2's unique filter failed and we cannot fall back to Assible's version "
"as it does not support the parameters supplied",
orig_exc=e)
