def _stats(parsed, key, statsd):
if isinstance(parsed, list):
working = []
for entry in parsed:
res, resstats = _stats(entry, key, statsd)
working.append(res)
statsd = dict_merge(statsd, resstats)
return working, statsd
if isinstance(parsed, dict):
working = {}
for k, val in parsed.items():
res, resstats = _stats(val, key, statsd)
working = dict_merge(working, {k: res})
statsd = dict_merge(statsd, resstats)
counts = None
if isinstance(res, dict):
try:
counts = dict(Counter(cval[key]
for ckey, cval in res.items()))
except (AttributeError, TypeError, KeyError):
pass
if isinstance(res, list):
try:
counts = dict(Counter(cval[key] for cval in res))
except (AttributeError, TypeError, KeyError):
pass
if counts:
stats_dict = {k + "_stats": {'count_by_' + key: counts,
'total': len(res)}}
working = dict_merge(working, stats_dict)
statsd = dict_merge(statsd, stats_dict)
return working, statsd
return parsed, statsd
def gen_config(self):
wantd = self.xform(self.want)
haved = self.xform(self.have)
# if state is merged, merge want onto have
if self.state == 'merged':
wantd = deepcopy(dict_merge(haved, wantd))
# validate the merged data through the argument_spec
validate_config(AclsArgs.argument_spec,
{"config": self.deform(deepcopy(wantd))})
# if state is deleted, limit the have to anything in want
# set want to nothing
if self.state == 'deleted':
haved = {k: v for k, v in haved.items() if k in wantd or not wantd}
wantd = {}
# handle everything common to want and have
for k, want in wantd.items():
self._compare_acl(want=want, have=haved.pop(k, {}))
# anything left should be deleted if deleted or overridden
if self.state in ['deleted', 'overridden']:
for k, have in haved.items():
self.addcmd(have, 'name', True)
def xform(self):
""" xform
"""
# convert lists of dicts into dicts, keyed on the uid of the obj
for thing in self.want, self.have:
thing['communities'] = {
entry['community']: entry
for entry in thing.get('communities', [])
}
thing['hosts'] = {(entry['host'], entry.get('udp_port')): entry
for entry in thing.get('hosts', [])}
thing['traps'] = {(trap['type'], name['name']): {
'name': name['name'],
'negate': not name['negate'],
'type': trap['type']
}
for trap in thing.get('traps', [])
for name in trap['names']}
thing['users'] = {(entry['username'], entry.get('engine_id')):
entry
for entry in thing.get('users', [])}
# if state is merged, merge want onto have
if self.state == 'merged':
self.want = dict_merge(self.have, self.want)
def _rekey_on_member_key(parsed, key):
if isinstance(parsed, dict):
working = {}
for k, val in parsed.items():
res = _rekey_on_member_key(val, key)
if isinstance(res, dict) and key in res:
if res[key] in working:
if not isinstance(working[res[key]], list):
working[res[key]] = [working[res[key]]]
working[res[key]].append(res)
else:
working[res[key]] = res
continue
working = dict_merge(working, {k: res})
return working
if isinstance(parsed, list):
if all(x.get(key) for x in parsed):
working = {}
for entry in parsed:
res = _rekey_on_member_key(entry, key)
if entry[key] in working:
for k, val in working.items():
if not isinstance(val, list):
working[k] = [val]
working[entry[key]].append(res)
else:
working[entry[key]] = res
return working
return parsed
def gen_config(self):
""" Select the appropriate function based on the state provided
:rtype: A list
:returns: the commands necessary to migrate the current configuration
to the desired configuration
"""
wantd = {entry['name']: entry for entry in self.want}
haved = {entry['name']: entry for entry in self.have}
# add a shutdown key for easier processing
for thing in wantd, haved:
for _name, entry in thing.items():
if entry.get('enable', None) is False:
entry['shutdown'] = True
# if state is merged, merge want onto have
if self.state == 'merged':
wantd = dict_merge(haved, wantd)
# if state is deleted, limit the have to anything in want
# set want to nothing
if self.state == 'deleted':
haved = {k: v for k, v in haved.items() if k in wantd or not wantd}
wantd = {}
# handle everything common to want and have
for k, want in wantd.items():
self._compare_interface(want=want, have=haved.pop(k, {}))
# anything left should be deleted if deleted or overridden
if self.state in ['deleted', 'overridden']:
for k, have in haved.items():
self._compare_interface(want={}, have=have)
def run(self, tmp=None, task_vars=None):
# pylint: disable=W0212
self._result = super(ActionModule, self).run(tmp, task_vars)
self._check_argspec()
self._set_vars(task_vars)
self._check_engine()
self._check_network_os()
self._check_for_errors()
self._load_filters()
self._check_transforms()
self._check_for_errors()
self._check_module_name()
if self._engine == 'pyats':
self._check_commands_against_pyats()
self._check_for_errors()
self._set_send_commands()
self._run_commands()
self._parse_stdout()
current_facts = task_vars.get('vars', {}).get('ansible_facts', {})
new_facts = dict_merge(current_facts, self._facts)
self._result.update({
'ansible_facts': new_facts,
'details': self._result_details
})
return self._result
def _add_facts(self, parsed):
if isinstance(parsed, list):
for chunk in parsed:
self._facts = dict_merge(self._facts, chunk)
elif isinstance(parsed, dict):
for k, val in parsed.items():
if k in self._facts:
# pylint: disable=C0123
if type(self._facts[k]) != type(val):
message = ("Cannot merge '{}' and '{}'"
" for facts key '{fkey}'. Ensure"
" all values for '{fkey}' are of"
" the same type".format(type(
self._facts[k]).__name__,
type(val).__name__,
fkey=k))
raise AnsibleError(message)
self._facts = dict_merge(self._facts, parsed)
def test_dict_merge():
base = dict(obj2=dict(),
b1=True,
b2=False,
b3=False,
one=1,
two=2,
three=3,
obj1=dict(key1=1, key2=2),
l1=[1, 3],
l2=[1, 2, 3],
l4=[4],
nested=dict(n1=dict(n2=2)))
other = dict(b1=True,
b2=False,
b3=True,
b4=True,
one=1,
three=4,
four=4,
obj1=dict(key1=2),
l1=[2, 1],
l2=[3, 2, 1],
l3=[1],
nested=dict(n1=dict(n2=2, n3=3)))
result = dict_merge(base, other)
# string assertions
assert 'one' in result
assert 'two' in result
assert result['three'] == 4
assert result['four'] == 4
# dict assertions
assert 'obj1' in result
assert 'key1' in result['obj1']
assert 'key2' in result['obj1']
# list assertions
assert result['l1'] == [1, 2, 3]
assert 'l2' in result
assert result['l3'] == [1]
assert 'l4' in result
# nested assertions
assert 'obj1' in result
assert result['obj1']['key1'] == 2
assert 'key2' in result['obj1']
# bool assertions
assert 'b1' in result
assert 'b2' in result
assert result['b3']
assert result['b4']
def test_dict_merge(self):
base = dict(obj2=dict(),
b1=True,
b2=False,
b3=False,
one=1,
two=2,
three=3,
obj1=dict(key1=1, key2=2),
l1=[1, 3],
l2=[1, 2, 3],
l4=[4],
nested=dict(n1=dict(n2=2)))
other = dict(b1=True,
b2=False,
b3=True,
b4=True,
one=1,
three=4,
four=4,
obj1=dict(key1=2),
l1=[2, 1],
l2=[3, 2, 1],
l3=[1],
nested=dict(n1=dict(n2=2, n3=3)))
result = dict_merge(base, other)
# string assertions
self.assertIn('one', result)
self.assertIn('two', result)
self.assertEqual(result['three'], 4)
self.assertEqual(result['four'], 4)
# dict assertions
self.assertIn('obj1', result)
self.assertIn('key1', result['obj1'])
self.assertIn('key2', result['obj1'])
# list assertions
self.assertEqual(result['l1'], [1, 2, 3])
self.assertIn('l2', result)
self.assertEqual(result['l3'], [1])
self.assertIn('l4', result)
# nested assertions
self.assertIn('obj1', result)
self.assertEqual(result['obj1']['key1'], 2)
self.assertIn('key2', result['obj1'])
# bool assertions
self.assertIn('b1', result)
self.assertIn('b2', result)
self.assertTrue(result['b3'])
self.assertTrue(result['b4'])
def _stats(parsed, key, statsd, missing_key="unknown"):
if isinstance(parsed, list):
working = []
for entry in parsed:
res, resstats = _stats(entry, key, statsd)
working.append(res)
statsd = dict_merge(statsd, resstats)
return working, statsd
if isinstance(parsed, dict):
working = {}
for k, val in parsed.items():
res, resstats = _stats(val, key, statsd)
working = dict_merge(working, {k: res})
statsd = dict_merge(statsd, resstats)
counts = None
if isinstance(res, dict):
try:
if any(key in cval for ckey, cval in res.items()):
counts = dict(
Counter(
str(cval.get(key, missing_key))
for ckey, cval in res.items()))
except (AttributeError, TypeError, KeyError):
pass
if isinstance(res, list):
try:
if any(key in cval for cval in res):
counts = dict(
Counter(
str(cval.get(key, missing_key))
for cval in res))
except (AttributeError, TypeError, KeyError):
pass
if counts:
ckey = 'count_by_' + key
count_dict = {ckey: counts}
count_dict[ckey]['total'] = len(res)
stats_dict = {"stats": count_dict}
working = dict_merge(working, stats_dict)
statsd = dict_merge(statsd, stats_dict)
return working, statsd
return parsed, statsd
def parse(self):
""" parse
"""
result = {}
shared = {}
for line in self._lines:
for parser in self._tmplt.PARSERS:
cap = re.match(parser['getval'], line)
if cap:
capdict = cap.groupdict()
capdict = {
k: v
for k, v in capdict.items() if v is not None
}
if parser.get('shared'):
shared = capdict
vals = dict_merge(capdict, shared)
res = self._deepformat(deepcopy(parser['result']), vals)
result = dict_merge(result, res)
break
return result
def _unnest(parsed, key):
if isinstance(parsed, list):
parsed = [_unnest(x, key) for x in parsed]
return parsed
if isinstance(parsed, dict):
working = {}
for k, val in parsed.items():
match = re.match(k, key)
if match and isinstance(val, dict):
working = dict_merge(working, _unnest(val, key))
else:
working[k] = _unnest(val, key)
return working
return parsed
def compare_arrays(old_params, new_params, param_name):
old = old_params.get(param_name) or []
new = new_params.get(param_name) or []
oldd = {}
for item in old:
name = item['name']
oldd[name] = item
newd = {}
for item in new:
name = item['name']
newd[name] = item
newd = dict_merge(oldd, newd)
return newd == oldd
def gen_config(self):
""" Select the appropriate function based on the state provided
:rtype: A list
:returns: the commands necessary to migrate the current configuration
to the desired configuration
"""
wantd = {(entry['id'], entry.get('vrf')): entry
for entry in self.want.get('processes', [])}
haved = {(entry['id'], entry.get('vrf')): entry
for entry in self.have.get('processes', [])}
# turn all lists of dicts into dicts prior to merge
for thing in wantd, haved:
for _pid, proc in thing.items():
for area in proc.get('areas', []):
area['ranges'] = {
entry['range']: entry
for entry in area.get('ranges', [])
}
proc['areas'] = {
entry['area']: entry
for entry in proc.get('areas', [])
}
# if state is merged, merge want onto have
if self.state == 'merged':
wantd = dict_merge(haved, wantd)
# if state is deleted, limit the have to anything in want
# set want to nothing
if self.state == 'deleted':
haved = {k: v for k, v in haved.items() if k in wantd or not wantd}
wantd = {}
# delete processes first so we do run into "more than one" errs
if self.state in ['overridden', 'deleted']:
for k, have in haved.items():
if k not in wantd:
self._compare(want={}, have=have)
self.addcmd(have, 'process_id', True)
for k, want in wantd.items():
self._compare(want=want, have=haved.pop(k, {}))
def test_dict_merge():
base = dict(obj2=dict(), b1=True, b2=False, b3=False,
one=1, two=2, three=3, obj1=dict(key1=1, key2=2),
l1=[1, 3], l2=[1, 2, 3], l4=[4],
nested=dict(n1=dict(n2=2)))
other = dict(b1=True, b2=False, b3=True, b4=True,
one=1, three=4, four=4, obj1=dict(key1=2),
l1=[2, 1], l2=[3, 2, 1], l3=[1],
nested=dict(n1=dict(n2=2, n3=3)))
result = dict_merge(base, other)
# string assertions
assert 'one' in result
assert 'two' in result
assert result['three'] == 4
assert result['four'] == 4
# dict assertions
assert 'obj1' in result
assert 'key1' in result['obj1']
assert 'key2' in result['obj1']
# list assertions
assert result['l1'] == [1, 2, 3]
assert 'l2' in result
assert result['l3'] == [1]
assert 'l4' in result
# nested assertions
assert 'obj1' in result
assert result['obj1']['key1'] == 2
assert 'key2' in result['obj1']
# bool assertions
assert 'b1' in result
assert 'b2' in result
assert result['b3']
assert result['b4']
def nxos_flatten_table_row(parsed, plural=False):
if isinstance(parsed, list):
parsed = [nxos_flatten_table_row(x, plural) for x in parsed]
return parsed
if isinstance(parsed, dict):
working = {}
for k, val in parsed.items():
if k.startswith("TABLE") or k.startswith("ROW"):
if isinstance(val, list):
name = "_".join(k.split('_')[1:])
if not name.endswith('s') and plural:
name += 's'
working[name] = [
nxos_flatten_table_row(x, plural) for x in val
]
elif isinstance(val, dict):
working = dict_merge(working,
nxos_flatten_table_row(val, plural))
else:
working[k] = nxos_flatten_table_row(val, plural)
return working
return parsed
def parse(self, source, template, tags):
templated_facts = {}
for entry in template:
name = entry.get('name')
entry_tags = entry.get('tags')
if tags and tags != entry_tags:
warning('skipping `%s` due to tagging' % name)
continue
facts = entry['facts']
matches = entry['matches']
loop = entry.get('loop')
section = entry.get('section')
context_data = list()
if section:
for item in self.sections:
if re.match(section, item, re.I):
block = self.network_config.get_block_config([item])
context_data.append(block)
else:
context_data.append(source)
for data in context_data:
variables = {'matches': list()}
for match in matches:
match_all = match.pop('match_all', False)
pattern = match['pattern']
match_var = match.get('match_var')
if match_all:
res = self.re_matchall(pattern, data)
else:
res = self.re_search(pattern, data)
if match_var:
variables[match_var] = res
variables['matches'].append(res)
#if when is not None:
# conditional = "{%% if %s %%}True{%% else %%}False{%% endif %%}"
# if not self.template(conditional % when, variables):
# display.vvvvv("context '%s' skipped due to conditional check failure" % name)
# continue
templated_obj = {}
if 'loop' in entry:
loop_data = self.template(entry['loop'], variables, convert_bare=True)
if loop_data:
for item in to_list(loop_data):
item_data = {'item': item}
obj = self.template(entry['facts'], item_data)
templated_facts = dict_merge(templated_facts, obj)
else:
obj = self.template(entry['facts'], variables)
templated_facts = dict_merge(templated_facts, obj)
return templated_facts
def _state_merged(self, want, have):
""" The command generator when state is merged
:rtype: A list
:returns: the commands necessary to merge the provided into
the current configuration
"""
commands = []
for want_afi in want.get("address_families", []):
have_afi = (self.find_af_context(want_afi,
have.get("address_families", []))
or {})
update_commands = []
for want_route in want_afi.get("routes", []):
have_route = (search_obj_in_list(
want_route["dest"], have_afi.get("routes", []), key="dest")
or {})
# convert the next_hops list of dictionaries to dictionary of
# dictionaries with (`dest_vrf`, `forward_router_address`, `interface`) tuple
# being the key for each dictionary.
# a combination of these 3 attributes uniquely identifies a route entry.
# in case `dest_vrf` is not specified, `forward_router_address` and `interface`
# become the unique identifier
rotated_have_next_hops = self.rotate_next_hops(
have_route.get("next_hops", {}))
rotated_want_next_hops = self.rotate_next_hops(
want_route.get("next_hops", {}))
# for every dict in the want next_hops dictionaries, if the key
# is present in `rotated_have_next_hops`, we set `existing` to True,
# which means the the given want exit point exists and we run dict_diff
# on `value` which is basically all the other attributes of the exit point
# if the key is not present, it means that this is a new exit point
for key, value in iteritems(rotated_want_next_hops):
if key in rotated_have_next_hops:
existing = True
have_exit_point_attribs = rotated_have_next_hops[key]
else:
existing = False
have_exit_point_attribs = {}
updates = dict_diff(have_exit_point_attribs, value)
if updates or not existing:
update_commands.append(
self._compute_commands(
dest=want_route["dest"],
next_hop=key,
# dict_merge() is necessary to make sure that we
# don't end up overridding the entry and also to
# allow incremental updates
updates=dict_merge(
rotated_have_next_hops.get(key, {}),
updates),
))
if update_commands:
update_commands.insert(
0,
"address-family {0} {1}".format(want_afi["afi"],
want_afi["safi"]),
)
commands.extend(update_commands)
if "vrf" in want and update_commands:
commands.insert(0, "vrf {0}".format(want["vrf"]))
return commands
