def test_service(service, device_type, tmp_path): svcstr = _get_service_def(service, device_type) assert svcstr sample_input = _get_service_data(service, device_type) assert sample_input created_records = cons_recs_from_json_template(svcstr, json.loads(sample_input)) assert created_records assert len(created_records) > 0 # this is the code necessary to write out the data file = tmp_path / f"{service}-{device_type}.yml" print(f"writing to {file}") file.write_text(yaml.dump(created_records)) processed_records = _get_processed_data(service, device_type) assert processed_records == created_records
if not isinstance(cmdstr, list): cmdstr = [svcdef.get('apply', {}).get(userargs.device_type, {})] isList = False raw_input = yml_inp.get('input', {}) \ .get(userargs.device_type, '') if not raw_input: print('No normalization service string found for {} in {}/{}'.format( userargs.device_type, userargs.sample_dir, userargs.service)) sys.exit(1) input_data = json.loads(raw_input) for i, cmd in enumerate(cmdstr): svcstr = cmd.get('normalize', '') if not svcstr: print( 'No normalization service string found for {} in {}/{}'.format( userargs.device_type, userargs.service_dir, userargs.service)) sys.exit(1) if isList: records = cons_recs_from_json_template(svcstr, input_data[i]) else: records = cons_recs_from_json_template(svcstr, input_data) print(json.dumps(records, indent=4))
sys.exit(1) with open('{}/{}.yml'.format(userargs.service_dir, userargs.service), 'r') as f: svcdef = yaml.safe_load(f.read()) with open('{}/{}.yml'.format(userargs.sample_dir, userargs.service), 'r') as f: yml_inp = yaml.safe_load(f.read()) # Extract the appropriate svc definition svcstr = svcdef.get('apply', {}) \ .get(userargs.device_type, {}) \ .get('normalize', '') if not svcstr: print('No normalization service string found for {} in {}/{}'.format( userargs.device_type, userargs.service_dir, userargs.service)) sys.exit(1) raw_input = yml_inp.get('input', {}) \ .get(userargs.device_type, '') if not raw_input: print('No normalization service string found for {} in {}/{}'.format( userargs.device_type, userargs.sample_dir, userargs.service)) sys.exit(1) records = cons_recs_from_json_template(svcstr, json.loads(raw_input)) print(json.dumps(records))
def _process_each_output(self, elem_num, data): """Workhorse processing routine for each element in output""" result = [] if (Service.is_status_ok(data.get('status', -1))): if not data["data"]: return result # Check host-specific normalization if any nfn = self.defn.get(data.get("hostname"), None) if not nfn: nfn = self.defn.get(data.get("devtype"), None) if nfn: # If we're riding on the coattails of another device # get that device's normalization function copynfn = nfn.get("copy", None) if copynfn: nfn = self.defn.get(copynfn, {}) norm_str = nfn.get("normalize", None) if norm_str is None and isinstance(nfn.get('command', None), list): # Can happen because we've a list of commands associated # Pick the normalization function associated with this # output norm_str = nfn.get('command', [])[elem_num].get('normalize', None) if norm_str: if isinstance(data["data"], str): try: in_info = json.loads(data["data"]) except json.JSONDecodeError: in_info = self.clean_json_input(data) if not in_info: self.logger.error( "Received non-JSON output where " "JSON was expected for {} on " "node {}, {}".format( data["cmd"], data["hostname"], data["data"])) return result try: in_info = json.loads(in_info) except json.JSONDecodeError: self.logger.error( "Received non-JSON output where " "JSON was expected for {} on " "node {}, {}".format( data["cmd"], data["hostname"], data["data"])) return result else: in_info = data["data"] if in_info: # EOS' HTTP returns errors like this. if isinstance(in_info, dict): tmp = in_info.get('data', []) if tmp and tmp[0].get('errors', {}): return [] result = cons_recs_from_json_template( norm_str, in_info) else: tfsm_template = nfn.get("textfsm", None) if tfsm_template is None and elem_num: # Can happen because we've a list of cmds associated # Pick the normalization function associated with this # output tfsm_template = nfn.get('command', [])[elem_num].get( 'textfsm', None) if not tfsm_template: return result in_info = [] if isinstance(data['data'], dict): if "output" in data["data"]: in_info = data["data"]["output"] elif "messages" in data["data"]: # This is Arista's bash output format in_info = data["data"]["messages"][0] if not in_info: in_info = data["data"] # Clean data is invoked inside this due to the way we # munge the data and force the types to adhere to the # specified type result = self.textfsm_data(in_info, tfsm_template, self.schema, data) else: self.logger.error( "{}: No normalization/textfsm function for device {}". format(self.name, data["hostname"])) return result
def process_data(self, data): """Derive the data to be stored from the raw input""" result = [] if (data["status"] is not None) and (int(data["status"]) == 200 or int(data["status"]) == 0): if not data["data"]: return result nfn = self.defn.get(data.get("hostname"), None) if not nfn: nfn = self.defn.get(data.get("devtype"), None) if nfn: copynfn = nfn.get("copy", None) if copynfn: nfn = self.defn.get(copynfn, {}) if nfn.get("normalize", None): if isinstance(data["data"], str): try: in_info = json.loads(data["data"]) except json.JSONDecodeError: in_info = self.clean_json_input(data) if not in_info: self.logger.error( "Received non-JSON output where " "JSON was expected for {} on " "node {}, {}".format( data["cmd"], data["hostname"], data["data"] ) ) return result try: in_info = json.loads(in_info) except json.JSONDecodeError: self.logger.error( "Received non-JSON output where " "JSON was expected for {} on " "node {}, {}".format( data["cmd"], data["hostname"], data["data"] ) ) return result else: in_info = data["data"] if in_info: # EOS' HTTP returns errors like this. if isinstance(in_info, dict): tmp = in_info.get('data', []) if tmp and tmp[0].get('errors', {}): return [] result = cons_recs_from_json_template( nfn.get("normalize", ""), in_info) result = self.clean_data(result, data) else: tfsm_template = nfn.get("textfsm", None) if not tfsm_template: return result if "output" in data["data"]: in_info = data["data"]["output"] elif "messages" in data["data"]: # This is Arista's bash output format in_info = data["data"]["messages"][0] else: in_info = data["data"] # Clean data is invoked inside this due to the way we # munge the data and force the types to adhere to the # specified type result = self.textfsm_data( in_info, tfsm_template, self.schema, data ) else: self.logger.error( "{}: No normalization/textfsm function for device {}" .format(self.name, data["hostname"])) return result