def _stats(parsed, key, statsd): if isinstance(parsed, list): working = [] for entry in parsed: res, resstats = _stats(entry, key, statsd) working.append(res) statsd = dict_merge(statsd, resstats) return working, statsd if isinstance(parsed, dict): working = {} for k, val in parsed.items(): res, resstats = _stats(val, key, statsd) working = dict_merge(working, {k: res}) statsd = dict_merge(statsd, resstats) counts = None if isinstance(res, dict): try: counts = dict(Counter(cval[key] for ckey, cval in res.items())) except (AttributeError, TypeError, KeyError): pass if isinstance(res, list): try: counts = dict(Counter(cval[key] for cval in res)) except (AttributeError, TypeError, KeyError): pass if counts: stats_dict = {k + "_stats": {'count_by_' + key: counts, 'total': len(res)}} working = dict_merge(working, stats_dict) statsd = dict_merge(statsd, stats_dict) return working, statsd return parsed, statsd
def gen_config(self): wantd = self.xform(self.want) haved = self.xform(self.have) # if state is merged, merge want onto have if self.state == 'merged': wantd = deepcopy(dict_merge(haved, wantd)) # validate the merged data through the argument_spec validate_config(AclsArgs.argument_spec, {"config": self.deform(deepcopy(wantd))}) # if state is deleted, limit the have to anything in want # set want to nothing if self.state == 'deleted': haved = {k: v for k, v in haved.items() if k in wantd or not wantd} wantd = {} # handle everything common to want and have for k, want in wantd.items(): self._compare_acl(want=want, have=haved.pop(k, {})) # anything left should be deleted if deleted or overridden if self.state in ['deleted', 'overridden']: for k, have in haved.items(): self.addcmd(have, 'name', True)
def xform(self): """ xform """ # convert lists of dicts into dicts, keyed on the uid of the obj for thing in self.want, self.have: thing['communities'] = { entry['community']: entry for entry in thing.get('communities', []) } thing['hosts'] = {(entry['host'], entry.get('udp_port')): entry for entry in thing.get('hosts', [])} thing['traps'] = {(trap['type'], name['name']): { 'name': name['name'], 'negate': not name['negate'], 'type': trap['type'] } for trap in thing.get('traps', []) for name in trap['names']} thing['users'] = {(entry['username'], entry.get('engine_id')): entry for entry in thing.get('users', [])} # if state is merged, merge want onto have if self.state == 'merged': self.want = dict_merge(self.have, self.want)
def _rekey_on_member_key(parsed, key): if isinstance(parsed, dict): working = {} for k, val in parsed.items(): res = _rekey_on_member_key(val, key) if isinstance(res, dict) and key in res: if res[key] in working: if not isinstance(working[res[key]], list): working[res[key]] = [working[res[key]]] working[res[key]].append(res) else: working[res[key]] = res continue working = dict_merge(working, {k: res}) return working if isinstance(parsed, list): if all(x.get(key) for x in parsed): working = {} for entry in parsed: res = _rekey_on_member_key(entry, key) if entry[key] in working: for k, val in working.items(): if not isinstance(val, list): working[k] = [val] working[entry[key]].append(res) else: working[entry[key]] = res return working return parsed
def gen_config(self): """ Select the appropriate function based on the state provided :rtype: A list :returns: the commands necessary to migrate the current configuration to the desired configuration """ wantd = {entry['name']: entry for entry in self.want} haved = {entry['name']: entry for entry in self.have} # add a shutdown key for easier processing for thing in wantd, haved: for _name, entry in thing.items(): if entry.get('enable', None) is False: entry['shutdown'] = True # if state is merged, merge want onto have if self.state == 'merged': wantd = dict_merge(haved, wantd) # if state is deleted, limit the have to anything in want # set want to nothing if self.state == 'deleted': haved = {k: v for k, v in haved.items() if k in wantd or not wantd} wantd = {} # handle everything common to want and have for k, want in wantd.items(): self._compare_interface(want=want, have=haved.pop(k, {})) # anything left should be deleted if deleted or overridden if self.state in ['deleted', 'overridden']: for k, have in haved.items(): self._compare_interface(want={}, have=have)
def run(self, tmp=None, task_vars=None): # pylint: disable=W0212 self._result = super(ActionModule, self).run(tmp, task_vars) self._check_argspec() self._set_vars(task_vars) self._check_engine() self._check_network_os() self._check_for_errors() self._load_filters() self._check_transforms() self._check_for_errors() self._check_module_name() if self._engine == 'pyats': self._check_commands_against_pyats() self._check_for_errors() self._set_send_commands() self._run_commands() self._parse_stdout() current_facts = task_vars.get('vars', {}).get('ansible_facts', {}) new_facts = dict_merge(current_facts, self._facts) self._result.update({ 'ansible_facts': new_facts, 'details': self._result_details }) return self._result
def _add_facts(self, parsed): if isinstance(parsed, list): for chunk in parsed: self._facts = dict_merge(self._facts, chunk) elif isinstance(parsed, dict): for k, val in parsed.items(): if k in self._facts: # pylint: disable=C0123 if type(self._facts[k]) != type(val): message = ("Cannot merge '{}' and '{}'" " for facts key '{fkey}'. Ensure" " all values for '{fkey}' are of" " the same type".format(type( self._facts[k]).__name__, type(val).__name__, fkey=k)) raise AnsibleError(message) self._facts = dict_merge(self._facts, parsed)
def test_dict_merge(): base = dict(obj2=dict(), b1=True, b2=False, b3=False, one=1, two=2, three=3, obj1=dict(key1=1, key2=2), l1=[1, 3], l2=[1, 2, 3], l4=[4], nested=dict(n1=dict(n2=2))) other = dict(b1=True, b2=False, b3=True, b4=True, one=1, three=4, four=4, obj1=dict(key1=2), l1=[2, 1], l2=[3, 2, 1], l3=[1], nested=dict(n1=dict(n2=2, n3=3))) result = dict_merge(base, other) # string assertions assert 'one' in result assert 'two' in result assert result['three'] == 4 assert result['four'] == 4 # dict assertions assert 'obj1' in result assert 'key1' in result['obj1'] assert 'key2' in result['obj1'] # list assertions assert result['l1'] == [1, 2, 3] assert 'l2' in result assert result['l3'] == [1] assert 'l4' in result # nested assertions assert 'obj1' in result assert result['obj1']['key1'] == 2 assert 'key2' in result['obj1'] # bool assertions assert 'b1' in result assert 'b2' in result assert result['b3'] assert result['b4']
def test_dict_merge(self): base = dict(obj2=dict(), b1=True, b2=False, b3=False, one=1, two=2, three=3, obj1=dict(key1=1, key2=2), l1=[1, 3], l2=[1, 2, 3], l4=[4], nested=dict(n1=dict(n2=2))) other = dict(b1=True, b2=False, b3=True, b4=True, one=1, three=4, four=4, obj1=dict(key1=2), l1=[2, 1], l2=[3, 2, 1], l3=[1], nested=dict(n1=dict(n2=2, n3=3))) result = dict_merge(base, other) # string assertions self.assertIn('one', result) self.assertIn('two', result) self.assertEqual(result['three'], 4) self.assertEqual(result['four'], 4) # dict assertions self.assertIn('obj1', result) self.assertIn('key1', result['obj1']) self.assertIn('key2', result['obj1']) # list assertions self.assertEqual(result['l1'], [1, 2, 3]) self.assertIn('l2', result) self.assertEqual(result['l3'], [1]) self.assertIn('l4', result) # nested assertions self.assertIn('obj1', result) self.assertEqual(result['obj1']['key1'], 2) self.assertIn('key2', result['obj1']) # bool assertions self.assertIn('b1', result) self.assertIn('b2', result) self.assertTrue(result['b3']) self.assertTrue(result['b4'])
def _stats(parsed, key, statsd, missing_key="unknown"): if isinstance(parsed, list): working = [] for entry in parsed: res, resstats = _stats(entry, key, statsd) working.append(res) statsd = dict_merge(statsd, resstats) return working, statsd if isinstance(parsed, dict): working = {} for k, val in parsed.items(): res, resstats = _stats(val, key, statsd) working = dict_merge(working, {k: res}) statsd = dict_merge(statsd, resstats) counts = None if isinstance(res, dict): try: if any(key in cval for ckey, cval in res.items()): counts = dict( Counter( str(cval.get(key, missing_key)) for ckey, cval in res.items())) except (AttributeError, TypeError, KeyError): pass if isinstance(res, list): try: if any(key in cval for cval in res): counts = dict( Counter( str(cval.get(key, missing_key)) for cval in res)) except (AttributeError, TypeError, KeyError): pass if counts: ckey = 'count_by_' + key count_dict = {ckey: counts} count_dict[ckey]['total'] = len(res) stats_dict = {"stats": count_dict} working = dict_merge(working, stats_dict) statsd = dict_merge(statsd, stats_dict) return working, statsd return parsed, statsd
def parse(self): """ parse """ result = {} shared = {} for line in self._lines: for parser in self._tmplt.PARSERS: cap = re.match(parser['getval'], line) if cap: capdict = cap.groupdict() capdict = { k: v for k, v in capdict.items() if v is not None } if parser.get('shared'): shared = capdict vals = dict_merge(capdict, shared) res = self._deepformat(deepcopy(parser['result']), vals) result = dict_merge(result, res) break return result
def _unnest(parsed, key): if isinstance(parsed, list): parsed = [_unnest(x, key) for x in parsed] return parsed if isinstance(parsed, dict): working = {} for k, val in parsed.items(): match = re.match(k, key) if match and isinstance(val, dict): working = dict_merge(working, _unnest(val, key)) else: working[k] = _unnest(val, key) return working return parsed
def compare_arrays(old_params, new_params, param_name): old = old_params.get(param_name) or [] new = new_params.get(param_name) or [] oldd = {} for item in old: name = item['name'] oldd[name] = item newd = {} for item in new: name = item['name'] newd[name] = item newd = dict_merge(oldd, newd) return newd == oldd
def gen_config(self): """ Select the appropriate function based on the state provided :rtype: A list :returns: the commands necessary to migrate the current configuration to the desired configuration """ wantd = {(entry['id'], entry.get('vrf')): entry for entry in self.want.get('processes', [])} haved = {(entry['id'], entry.get('vrf')): entry for entry in self.have.get('processes', [])} # turn all lists of dicts into dicts prior to merge for thing in wantd, haved: for _pid, proc in thing.items(): for area in proc.get('areas', []): area['ranges'] = { entry['range']: entry for entry in area.get('ranges', []) } proc['areas'] = { entry['area']: entry for entry in proc.get('areas', []) } # if state is merged, merge want onto have if self.state == 'merged': wantd = dict_merge(haved, wantd) # if state is deleted, limit the have to anything in want # set want to nothing if self.state == 'deleted': haved = {k: v for k, v in haved.items() if k in wantd or not wantd} wantd = {} # delete processes first so we do run into "more than one" errs if self.state in ['overridden', 'deleted']: for k, have in haved.items(): if k not in wantd: self._compare(want={}, have=have) self.addcmd(have, 'process_id', True) for k, want in wantd.items(): self._compare(want=want, have=haved.pop(k, {}))
def test_dict_merge(): base = dict(obj2=dict(), b1=True, b2=False, b3=False, one=1, two=2, three=3, obj1=dict(key1=1, key2=2), l1=[1, 3], l2=[1, 2, 3], l4=[4], nested=dict(n1=dict(n2=2))) other = dict(b1=True, b2=False, b3=True, b4=True, one=1, three=4, four=4, obj1=dict(key1=2), l1=[2, 1], l2=[3, 2, 1], l3=[1], nested=dict(n1=dict(n2=2, n3=3))) result = dict_merge(base, other) # string assertions assert 'one' in result assert 'two' in result assert result['three'] == 4 assert result['four'] == 4 # dict assertions assert 'obj1' in result assert 'key1' in result['obj1'] assert 'key2' in result['obj1'] # list assertions assert result['l1'] == [1, 2, 3] assert 'l2' in result assert result['l3'] == [1] assert 'l4' in result # nested assertions assert 'obj1' in result assert result['obj1']['key1'] == 2 assert 'key2' in result['obj1'] # bool assertions assert 'b1' in result assert 'b2' in result assert result['b3'] assert result['b4']
def nxos_flatten_table_row(parsed, plural=False): if isinstance(parsed, list): parsed = [nxos_flatten_table_row(x, plural) for x in parsed] return parsed if isinstance(parsed, dict): working = {} for k, val in parsed.items(): if k.startswith("TABLE") or k.startswith("ROW"): if isinstance(val, list): name = "_".join(k.split('_')[1:]) if not name.endswith('s') and plural: name += 's' working[name] = [ nxos_flatten_table_row(x, plural) for x in val ] elif isinstance(val, dict): working = dict_merge(working, nxos_flatten_table_row(val, plural)) else: working[k] = nxos_flatten_table_row(val, plural) return working return parsed
def parse(self, source, template, tags): templated_facts = {} for entry in template: name = entry.get('name') entry_tags = entry.get('tags') if tags and tags != entry_tags: warning('skipping `%s` due to tagging' % name) continue facts = entry['facts'] matches = entry['matches'] loop = entry.get('loop') section = entry.get('section') context_data = list() if section: for item in self.sections: if re.match(section, item, re.I): block = self.network_config.get_block_config([item]) context_data.append(block) else: context_data.append(source) for data in context_data: variables = {'matches': list()} for match in matches: match_all = match.pop('match_all', False) pattern = match['pattern'] match_var = match.get('match_var') if match_all: res = self.re_matchall(pattern, data) else: res = self.re_search(pattern, data) if match_var: variables[match_var] = res variables['matches'].append(res) #if when is not None: # conditional = "{%% if %s %%}True{%% else %%}False{%% endif %%}" # if not self.template(conditional % when, variables): # display.vvvvv("context '%s' skipped due to conditional check failure" % name) # continue templated_obj = {} if 'loop' in entry: loop_data = self.template(entry['loop'], variables, convert_bare=True) if loop_data: for item in to_list(loop_data): item_data = {'item': item} obj = self.template(entry['facts'], item_data) templated_facts = dict_merge(templated_facts, obj) else: obj = self.template(entry['facts'], variables) templated_facts = dict_merge(templated_facts, obj) return templated_facts
def _state_merged(self, want, have): """ The command generator when state is merged :rtype: A list :returns: the commands necessary to merge the provided into the current configuration """ commands = [] for want_afi in want.get("address_families", []): have_afi = (self.find_af_context(want_afi, have.get("address_families", [])) or {}) update_commands = [] for want_route in want_afi.get("routes", []): have_route = (search_obj_in_list( want_route["dest"], have_afi.get("routes", []), key="dest") or {}) # convert the next_hops list of dictionaries to dictionary of # dictionaries with (`dest_vrf`, `forward_router_address`, `interface`) tuple # being the key for each dictionary. # a combination of these 3 attributes uniquely identifies a route entry. # in case `dest_vrf` is not specified, `forward_router_address` and `interface` # become the unique identifier rotated_have_next_hops = self.rotate_next_hops( have_route.get("next_hops", {})) rotated_want_next_hops = self.rotate_next_hops( want_route.get("next_hops", {})) # for every dict in the want next_hops dictionaries, if the key # is present in `rotated_have_next_hops`, we set `existing` to True, # which means the the given want exit point exists and we run dict_diff # on `value` which is basically all the other attributes of the exit point # if the key is not present, it means that this is a new exit point for key, value in iteritems(rotated_want_next_hops): if key in rotated_have_next_hops: existing = True have_exit_point_attribs = rotated_have_next_hops[key] else: existing = False have_exit_point_attribs = {} updates = dict_diff(have_exit_point_attribs, value) if updates or not existing: update_commands.append( self._compute_commands( dest=want_route["dest"], next_hop=key, # dict_merge() is necessary to make sure that we # don't end up overridding the entry and also to # allow incremental updates updates=dict_merge( rotated_have_next_hops.get(key, {}), updates), )) if update_commands: update_commands.insert( 0, "address-family {0} {1}".format(want_afi["afi"], want_afi["safi"]), ) commands.extend(update_commands) if "vrf" in want and update_commands: commands.insert(0, "vrf {0}".format(want["vrf"])) return commands