def test_service(service, device_type, tmp_path):
svcstr = _get_service_def(service, device_type)
assert svcstr
sample_input = _get_service_data(service, device_type)
assert sample_input
created_records = cons_recs_from_json_template(svcstr, json.loads(sample_input))
assert created_records
assert len(created_records) > 0
# this is the code necessary to write out the data
file = tmp_path / f"{service}-{device_type}.yml"
print(f"writing to {file}")
file.write_text(yaml.dump(created_records))
processed_records = _get_processed_data(service, device_type)
assert processed_records == created_records
if not isinstance(cmdstr, list):
cmdstr = [svcdef.get('apply', {}).get(userargs.device_type, {})]
isList = False
raw_input = yml_inp.get('input', {}) \
.get(userargs.device_type, '')
if not raw_input:
print('No normalization service string found for {} in {}/{}'.format(
userargs.device_type, userargs.sample_dir, userargs.service))
sys.exit(1)
input_data = json.loads(raw_input)
for i, cmd in enumerate(cmdstr):
svcstr = cmd.get('normalize', '')
if not svcstr:
print(
'No normalization service string found for {} in {}/{}'.format(
userargs.device_type, userargs.service_dir,
userargs.service))
sys.exit(1)
if isList:
records = cons_recs_from_json_template(svcstr, input_data[i])
else:
records = cons_recs_from_json_template(svcstr, input_data)
print(json.dumps(records, indent=4))
sys.exit(1)
with open('{}/{}.yml'.format(userargs.service_dir, userargs.service), 'r') as f:
svcdef = yaml.safe_load(f.read())
with open('{}/{}.yml'.format(userargs.sample_dir, userargs.service), 'r') as f:
yml_inp = yaml.safe_load(f.read())
# Extract the appropriate svc definition
svcstr = svcdef.get('apply', {}) \
.get(userargs.device_type, {}) \
.get('normalize', '')
if not svcstr:
print('No normalization service string found for {} in {}/{}'.format(
userargs.device_type, userargs.service_dir, userargs.service))
sys.exit(1)
raw_input = yml_inp.get('input', {}) \
.get(userargs.device_type, '')
if not raw_input:
print('No normalization service string found for {} in {}/{}'.format(
userargs.device_type, userargs.sample_dir, userargs.service))
sys.exit(1)
records = cons_recs_from_json_template(svcstr, json.loads(raw_input))
print(json.dumps(records))
def _process_each_output(self, elem_num, data):
"""Workhorse processing routine for each element in output"""
result = []
if (Service.is_status_ok(data.get('status', -1))):
if not data["data"]:
return result
# Check host-specific normalization if any
nfn = self.defn.get(data.get("hostname"), None)
if not nfn:
nfn = self.defn.get(data.get("devtype"), None)
if nfn:
# If we're riding on the coattails of another device
# get that device's normalization function
copynfn = nfn.get("copy", None)
if copynfn:
nfn = self.defn.get(copynfn, {})
norm_str = nfn.get("normalize", None)
if norm_str is None and isinstance(nfn.get('command', None),
list):
# Can happen because we've a list of commands associated
# Pick the normalization function associated with this
# output
norm_str = nfn.get('command',
[])[elem_num].get('normalize', None)
if norm_str:
if isinstance(data["data"], str):
try:
in_info = json.loads(data["data"])
except json.JSONDecodeError:
in_info = self.clean_json_input(data)
if not in_info:
self.logger.error(
"Received non-JSON output where "
"JSON was expected for {} on "
"node {}, {}".format(
data["cmd"], data["hostname"],
data["data"]))
return result
try:
in_info = json.loads(in_info)
except json.JSONDecodeError:
self.logger.error(
"Received non-JSON output where "
"JSON was expected for {} on "
"node {}, {}".format(
data["cmd"], data["hostname"],
data["data"]))
return result
else:
in_info = data["data"]
if in_info:
# EOS' HTTP returns errors like this.
if isinstance(in_info, dict):
tmp = in_info.get('data', [])
if tmp and tmp[0].get('errors', {}):
return []
result = cons_recs_from_json_template(
norm_str, in_info)
else:
tfsm_template = nfn.get("textfsm", None)
if tfsm_template is None and elem_num:
# Can happen because we've a list of cmds associated
# Pick the normalization function associated with this
# output
tfsm_template = nfn.get('command', [])[elem_num].get(
'textfsm', None)
if not tfsm_template:
return result
in_info = []
if isinstance(data['data'], dict):
if "output" in data["data"]:
in_info = data["data"]["output"]
elif "messages" in data["data"]:
# This is Arista's bash output format
in_info = data["data"]["messages"][0]
if not in_info:
in_info = data["data"]
# Clean data is invoked inside this due to the way we
# munge the data and force the types to adhere to the
# specified type
result = self.textfsm_data(in_info, tfsm_template,
self.schema, data)
else:
self.logger.error(
"{}: No normalization/textfsm function for device {}".
format(self.name, data["hostname"]))
return result
def process_data(self, data):
"""Derive the data to be stored from the raw input"""
result = []
if (data["status"] is not None) and (int(data["status"]) == 200 or
int(data["status"]) == 0):
if not data["data"]:
return result
nfn = self.defn.get(data.get("hostname"), None)
if not nfn:
nfn = self.defn.get(data.get("devtype"), None)
if nfn:
copynfn = nfn.get("copy", None)
if copynfn:
nfn = self.defn.get(copynfn, {})
if nfn.get("normalize", None):
if isinstance(data["data"], str):
try:
in_info = json.loads(data["data"])
except json.JSONDecodeError:
in_info = self.clean_json_input(data)
if not in_info:
self.logger.error(
"Received non-JSON output where "
"JSON was expected for {} on "
"node {}, {}".format(
data["cmd"], data["hostname"],
data["data"]
)
)
return result
try:
in_info = json.loads(in_info)
except json.JSONDecodeError:
self.logger.error(
"Received non-JSON output where "
"JSON was expected for {} on "
"node {}, {}".format(
data["cmd"], data["hostname"],
data["data"]
)
)
return result
else:
in_info = data["data"]
if in_info:
# EOS' HTTP returns errors like this.
if isinstance(in_info, dict):
tmp = in_info.get('data', [])
if tmp and tmp[0].get('errors', {}):
return []
result = cons_recs_from_json_template(
nfn.get("normalize", ""), in_info)
result = self.clean_data(result, data)
else:
tfsm_template = nfn.get("textfsm", None)
if not tfsm_template:
return result
if "output" in data["data"]:
in_info = data["data"]["output"]
elif "messages" in data["data"]:
# This is Arista's bash output format
in_info = data["data"]["messages"][0]
else:
in_info = data["data"]
# Clean data is invoked inside this due to the way we
# munge the data and force the types to adhere to the
# specified type
result = self.textfsm_data(
in_info, tfsm_template, self.schema, data
)
else:
self.logger.error(
"{}: No normalization/textfsm function for device {}"
.format(self.name, data["hostname"]))
return result
