def query_node_info(self): """ If the router name is known, but the nodes are not or the nodes are known, but the asset ids are not Send a GraphQL query to get the missing information... """ if self.gql_token is None: return if self.have_asset_info() and \ self.have_active_info(): return qr = gql_helper.NodeGQL("allRouters", ['name'], [self.get_router()], debug=self.debug) qn = gql_helper.NodeGQL("nodes", [ 'name', 'assetId', 'state { processes { name status primary leaderStatus } }' ]) qr.add_node(qn) json_reply = {} api_errors = [] query_status = qr.send_query(self.gql_token, json_reply, api_errors) if query_status != 200 or \ len(api_errors) > 0: return match_string = "node.state.processes[]" merged_list = self._format_allRouters_reply(json_reply, match_string) if self.debug: print('........ flattened list ..........') pprint.pprint(merged_list) for entry in merged_list: if self.get_asset_by_type(entry['node_type'], load=False) is None: # create an asset enty with data available asset = {} asset['assetId'] = entry['asset_id'] asset['nodeName'] = entry['node_name'] asset['t128Version'] = 'Unknown' self.update(asset) if entry['primary']: self.active_proc_map[entry['name']] = entry['node_name'] self.active_asset_map[entry['name']] = entry['asset_id']
def get_json_assets(self): """ Loads a list of json assets from the conductor """ qn = gql_helper.NodeGQL("allAuthorities", [ 'assets { assetId routerName nodeName t128Version status assetIdConfigured text failedStatus }' ], [ ], top_level=True, debug=self.debug) json_reply={} query_status = qn.send_query(self.gql_token, json_reply) if not query_status == 200: print('\n*** Unable to query conductor assets ***\n') sys.exit(1) match_string=f"node.assets[*]" flatter_json = jmespath.search(match_string, json_reply) if self.debug: print('........ flattened list ..........') pprint.pprint(flatter_json) self.json_assets = flatter_json return query_status
def run(self, local_info, router_context, gql_token, fp): """ This test usess the gql engine to get node connectivity status """ test_info = self.test_info(local_info, router_context) self.output.test_start(test_info) params = self.get_params() exclude_list = params["exclude_tests"] expected_entries = params["expected_entries"] qr = gql_helper.NodeGQL("allRouters", ['name'], [router_context.get_router()], debug=self.debug) qn = gql_helper.NodeGQL("nodes", [ 'name', 'connectivity { status remoteRouterName remoteNodeName remoteNodeRole}' ]) qr.add_node(qn) json_reply = {} query_status = AbstractTest.GraphQL.ReplyStatus() if not self.send_query(qr, gql_token, json_reply, query_status): return self.output.test_end(fp) flatter_json = qr.flatten_json(json_reply, 'router/nodes/connectivity', '/') router_context.set_allRouters_node_type(flatter_json) if self.debug: print('........ flattened list ..........') pprint.pprint(flatter_json) entry_count = 0 fail_count = 0 for entry in flatter_json: entry_count += 1 if entry is None: # In the past this happens when a node fails to register the 'show # system connectivity' command for itself self.output.proc_empty_reply(status=Output.Status.FAIL) fail_count += 1 continue if self.exclude_flat_entry(entry, exclude_list): continue self.output.proc_node_status_mismatch(entry) fail_count += 1 expected_entries = params["expected_entries"] status = self.output.status if query_status.server_code == 200: if entry_count < expected_entries: self.output.proc_test_result_too_few(entry_count, fail_count, params) elif status == Output.Status.FAIL: self.output.proc_test_result_wrong_state( fail_count, expected_entries, params) else: self.output.proc_test_result_all_ok(entry_count, expected_entries, params) else: self.output.proc_test_result_bad_query(query_status.server_code) return self.output.test_end(fp)
def run(self, local_info, router_context, gql_token, fp): """ This test uses the gql engine to get asset state This query only works on the conductor (why? the conductor learns asset state from the managed nodes so why show it only on the conductor? This test will have to include a local node role test, which can be checked for 'conductor' and return status WARN otherwise... """ test_info = self.test_info(local_info, router_context) self.output.test_start(test_info, status=Output.Status.OK) params = self.get_params() exclude_tests = params["exclude_tests"] entry_tests = params["entry_tests"] no_match = {} if "no_match" in entry_tests: no_match = entry_tests["no_match"] if local_info.get_node_type() != 'conductor': self.output.unsupported_node_type(local_info) return self.output.test_end(fp) qn = gql_helper.NodeGQL("allAuthorities", [ 'assets { assetId routerName nodeName t128Version status assetIdConfigured ' + 'text failedStatus }' ], [ ], top_level=True, debug=self.debug) json_reply={} if not self.send_query(qn, gql_token, json_reply): return self.output.test_end(fp) #flatter_json = self.flatten_json(json_reply, 'node/assets', '/', prefix='') match_string=f"node.assets[*] | [?routerName=='{router_context.get_router()}']" flatter_json = jmespath.search(match_string, json_reply) router_context.set_allRouters_node_type(flatter_json, node_name_key="nodeName") if self.debug: print('........ flattened list ..........') pprint.pprint(flatter_json) stats = {} Output.init_result_stats(stats) stats["total_count"] = len(flatter_json) engine = EntryTest.Parser() for entry in flatter_json: if engine.exclude_entry(entry, exclude_tests): stats["exclude_count"] += 1 continue test_status = engine.eval_entry_by_tests(entry, params["entry_tests"]) Output.update_stats(stats, test_status) if test_status != None: self.output.proc_interim_result(entry, status=test_status) # All pass versus fail decisions belong outside of the output module if stats["FAIL"] > 0: self.output.proc_test_result(entry_tests, stats, status=Output.Status.FAIL) else: self.output.proc_test_result(entry_tests, stats, status=Output.Status.OK) return self.output.test_end(fp)
def run(self, local_info, router_context, gql_token, fp): fib_table_fields=[ "totalCount" ] fib_route_fields=[ "serviceName " "route { " "ipPrefix " "l4Port " "l4PortUpper " "protocol " "tenant " "nextHops { devicePort gateway nodeId vlan deviceInterface networkInterface } " "}" ] test_info = self.test_info(local_info, router_context) self.output.test_start(test_info) params = self.get_params() exclude_tests = params["exclude_tests"] fib_entry_suffix='' if params["filter_string"] != "" or \ params["maximum_query_count"] > 0: fib_entry_suffix = '(' if params["maximum_query_count"] > 0: fib_entry_suffix += f"first: {params['maximum_query_count']}" if params["filter_string"] != "": if len(fib_entry_suffix) > 1: fib_entry_suffix += ',' fib_entry_suffix += f'filter: "\\"\\"~\\"{params["filter_string"]}\\""' fib_entry_suffix += ')' if local_info.get_router_name() == router_context.get_router() and \ local_info.get_node_type() == 'conductor': self.output.unsupported_node_type(local_info) return Output.Status.WARN fib_route_data = [] if len(params["entry_tests"]) > 0: fib_route_data = fib_route_fields node_names = [] if 'node_type' in params: node_name = router_context.get_node_by_type(params['node_type']) if node_name is not None and \ node_name != '': node_names.append(node_name) # This query is run against all nodes for the router qR = gql_helper.NodeGQL("allRouters", ["name"], [router_context.get_router()], debug=self.debug) qn = gql_helper.NodeGQL("nodes", ["name"], node_names) qf = gql_helper.NodeGQL(f"fibEntries{fib_entry_suffix}", fib_route_data) #qr = gql_helper.NodeGQL("route", fib_route_data) qR.add_node(qn) qn.add_node(qf) #qf.add_node(qr) json_reply={} if not self.send_query(qR, gql_token, json_reply): return self.output.test_end(fp) flatter_json = qR.flatten_json(json_reply, 'router/nodes/fibEntries/route', '/') if self.debug: print('........ flattened list ..........') pprint.pprint(flatter_json) stats = {} Output.init_result_stats(stats) stats["total_count"] = len(flatter_json) if params["minimum_threshold"] > 0 and \ total_count < params["minimumm_threshold"]: self.output.add_too_few_entries(params) return self.output.test_end(fp) if params["maximum_threshold"] > 0 and \ total_count > params["maximum_threshold"]: self.output.add_too_man_entries_parameters(params) return self.output.test_end(fp) engine = EntryTest.Parser(self.debug) for entry in flatter_json: if engine.exclude_entry(entry, exclude_tests): stats["exclude_count"] += 1 continue test_status = engine.eval_entry_by_tests( entry, params["entry_tests"] ) if test_status == Output.Status.FAIL: test.output.proc_test_match(entry) stats["fail_count"] += 1 if test_status == Output.Status.WARN: test.output.proc_test_match(entry) stats["warn_count"] += 1 self.output.proc_test_result(params, stats) return self.output.test_end(fp)
def run(self, local_info, router_context, gql_token, fp): """ This test uses the gql engine to get device state state """ # Process test parameters test_info = self.test_info(local_info, router_context) self.output.test_start(test_info) params = self.get_params() # TODO either keep and implement as a parameter or toss include_list = params["device-interfaces"] # GraophQL Preperation qr = gql_helper.NodeGQL("allRouters", ['name'], [router_context.get_router()], debug=self.debug) qn = gql_helper.NodeGQL("nodes", ['name']) qd = gql_helper.NodeGQL("deviceInterfaces", ['name', 'state { operationalStatus }'], include_list) qr.add_node(qn) qn.add_node(qd) json_reply = {} if not self.send_query(qr, gql_token, json_reply): return self.output.test_end(fp) flatter_json = qr.flatten_json(json_reply, 'router/nodes/deviceInterfaces', '/') router_context.set_allRouters_node_type(flatter_json) if self.debug: print('........ flattened list ..........') pprint.pprint(flatter_json) # Run test / process results fail_count = 0 warn_count = 0 engine = EntryTest.Parser() for entry in flatter_json: try: if self.exclude_flat_entry(entry, params["exclude_tests"]): continue except KeyError: pass test_status = engine.eval_entry_by_tests(entry, params["entry_tests"]) if test_status == Output.Status.FAIL: self.output.proc_test_match(test_status, entry) fail_count += 1 if test_status == Output.Status.WARN: self.output.proc_test_match(test_status, entry) warn_count += 1 if fail_count > 0: self.output.proc_test_fail_result(fail_count) elif warn_count > 0: self.output.proc_test_warn_result(warn_count) return self.output.test_end(fp)
def run(self, local_info, router_context, gql_token, fp): """ This test uses the gql engine to learn gateway IP addresses and ping them, processing the latency results. Note apparently timeout is in seconds and not ms as suggested by grapql documentation """ test_info = self.test_info(local_info, router_context) self.output.test_start(test_info) params = self.get_params() # First, query allRouters...nodes...networkInterfaces... to determine # the router and gateway """ API = allRouters Fields = name """ intf_list = params["network-interfaces"] dev_list = params["device-interfaces"] if self.debug: print(f'------ Network Interfaces to Process ------') pprint.pprint(intf_list) qr = gql_helper.NodeGQL("allRouters", ['name'], [router_context.get_router()], debug=self.debug) qn = gql_helper.NodeGQL("nodes", ['name', 'assetId']) qd = gql_helper.NodeGQL( "deviceInterfaces", ['name', 'sharedPhysAddress', 'state { operationalStatus }'], dev_list) qi = gql_helper.NodeGQL( "networkInterfaces", ['name', 'state { addresses { ipAddress gateway prefixLength } }'], intf_list) qa = gql_helper.NodeGQL("addresses", ['ipAddress', 'gateway', 'prefixLength']) qr.add_node(qn) qn.add_node(qd) qd.add_node(qi) qi.add_node(qa) json_reply = {} if not self.send_query(qr, gql_token, json_reply): return self.output.test_end(fp) if params["static-address"]: flatter_json = qr.flatten_json( json_reply, 'router/nodes/deviceInterfaces/networkInterfaces/addresses', '/') ni_name_key = 'router/nodes/deviceInterfaces/networkInterfaces/name' else: # stop prefix should be router/nodes/deviceInterfaces/networkInterfaces/state, but because no address is # returned for one of the peers (state=None), flatten_json skips that node. So go one level higher and # live with it for now. flatter_json = qr.flatten_json( json_reply, 'router/nodes/deviceInterfaces/networkInterfaces', '/') ni_name_key = 'name' router_context.set_allRouters_node_type(flatter_json) self._workaround_graphql_api(flatter_json) if self.debug: print('........ flattened list ..........') pprint.pprint(flatter_json) self.output.progress_start(fp) gateway_success_count = 0 gateway_fail_count = 0 gateway_count = 0 for entry in flatter_json: if self.debug: print(f'%%% process NI for Ping %%%') pprint.pprint(entry) if params["node_type"] != '' and \ entry['node_type'] != params["node_type"]: if self.debug: print( f"NI {entry[ni_name_key]}: Skipping {entry['node_type']} node {entry['node_name']}" ) continue address = '' gateway = '' try: if params["static-address"]: addresses = [entry] else: addresses = entry['state']['addresses'] if self.debug: print(f'%%%% process address for Ping %%%%') pprint.pprint(addresses) egress_interface = entry[ni_name_key] for address_entry in addresses: address = address_entry['ipAddress'] gateway = address_entry['gateway'] prefix_length = int(address_entry['prefixLength']) target = gateway dest_ip = None if gateway is None: # hack for ipv4 only! hack = int(ipaddress.IPv4Address(address)) if prefix_length == 31: gateway_hack = hack & 0xfffffffe | -(hack & 0x01) + 1 gateway_ip = ipaddress.IPv4Address(gateway_hack) dest_ip = str(gateway_ip) gateway = '' target = dest_ip elif prefix_length == 30: gateway_hack = hack & 0xfffffffc | -(hack & 0x03) + 3 gateway_ip = ipaddress.IPv4Address(gateway_hack) dest_ip = str(gateway_ip) gateway = '' target = dest_ip else: self.output.proc_cannot_ping_no_gateway( entry, ni_name_key) msg_list.append( f'Cannot ping via NI {entry[ni_name_key]}, No Gateway!' ) gateway_count += 1 continue try: oper_status = entry[ 'router/nodes/deviceInterfaces/state/operationalStatus'] if oper_status != 'OPER_UP': self.output.proc_cannot_ping_dev_status( entry, ni_name_key, oper_status) gateway_count += 1 #continue break except KeyError: # Continue as there is no operationalStatus to evaluate pass # Invoke Graphql PING API ping_count = 0 if dest_ip is None: if "destination-ip" in params and \ params["destination-ip"] != '': dest_ip = params["destination-ip"] else: dest_ip = gateway size = params["size"] timeout = params["timeout"] seqno = params["sequence"] router = entry["router/name"] node = entry["node_name"] identifier = params["identifier"] total_response_time = float(0) average_response_time = float(0) ping_success_count = 0 ping_fail_count = 0 while ping_count < params["iterations"]: argstr = f'routerName: "{router}"' argstr += f', nodeName: "{node}"' argstr += f', identifier: {identifier}' argstr += f', egressInterface: "{egress_interface}"' if dest_ip != '': argstr += f', destinationIp: "{dest_ip}"' if gateway != '': argstr += f', gatewayIp: "{gateway}"' argstr += f', sequence: {seqno}' if size != '': argstr += f', size: {size}' argstr += f', timeout: {timeout}' if self.debug: print(f'argstr={argstr}') # display progress in-place as does 128status.sh... now_message = f"NI {entry[ni_name_key]}: ping {gateway} {ping_count}/{params['iterations']} tmo={params['timeout']}s" self.output.progress_display(now_message, fp) qp = gql_helper.RawGQL( f'ping({argstr}) ' + '{ status statusReason reachable sequence ttl responseTime }', debug=self.debug) json_ping_reply = {} qp.send_query(gql_token, json_ping_reply) # standard graphql error processing may not be appropriate here as a failure can # be part of the test process w/o ruining the test. ping_count += 1 seqno += 1 try: # "0" used < 4.2.0; "SUCCESS" used in 4.2.0+ json_ping_reply = json_ping_reply['ping'] if json_ping_reply['status'] == "0" or \ json_ping_reply['status'] == "SUCCESS": ping_success_count += 1 total_response_time += float( json_ping_reply['responseTime']) average_response_time = total_response_time / float( ping_success_count) else: ping_fail_count += 1 except (KeyError, TypeError) as e: self.output.proc_no_data_in_reply( entry, ni_name_key, gateway) ping_fail_count += 1 gateway_count += 1 continue if ping_count == ping_success_count: # fix this for multiple matching entries gateway_success_count += 1 self.output.proc_ping_result_pass( entry, ni_name_key, ping_count, ping_success_count, target, average_response_time) else: gateway_fail_count += 1 self.output.proc_ping_result_fail( entry, ni_name_key, ping_count, ping_fail_count, target, average_response_time) gateway_count += 1 except (TypeError) as e: self.output.proc_no_address_in_reply(entry, ni_name_key) continue status = self.output.status if gateway_count == 0: status = Output.Status.WARN if gateway_count != gateway_success_count: status = Output.Status.FAIL self.output.proc_test_result(status, gateway_count, gateway_success_count, gateway_fail_count, params) return self.output.test_end(fp)
def run(self, local_info, router_context, gql_token, fp): flowEntryFields = [ \ 'sourceIp', 'destIp', 'sourcePort', 'destPort', 'vlan', 'devicePort', 'protocol', 'sessionUuid', 'natIp', 'natPort', 'serviceName', 'tenant', 'encrypted', 'inactivityTimeout', 'deviceInterfaceName', 'networkInterfaceName', 'startTime', 'forward' ] test_info = self.test_info(local_info, router_context) self.output.test_start(test_info) params = self.get_params() try: idle_threshold_seconds = params["idle_threshold_seconds"] idle_maximum_seconds = params["idle_maximum_seconds"] max_sessions = params["max_sessions_to_query"] filter_string = params["filter_string"] match_port = params["match_port"] except Exception as e: # TODO: Improve error handling print("CONFIG ERROR\n") return Output.Status.FAIL exclude_tests = [] if "exclude_tests" in params: exclude_tests = params["exclude_tests"] flow_entry_suffix = f'(first: {max_sessions}, filter: "\\"\\"~\\"{filter_string}\\"")' if local_info.get_router_name() == router_context.get_router() and \ local_info.get_node_type() == 'conductor': # Check Error output self.output.unsupported_node_type(local_info) return Output.Status.WARN qr = gql_helper.NodeGQL("allRouters", ['name'], [router_context.get_router()], debug=self.debug) qn = gql_helper.NodeGQL("nodes", ['name']) qf = gql_helper.NodeGQL(f"flowEntries{flow_entry_suffix}", flowEntryFields) qr.add_node(qn) qn.add_node(qf) json_reply = {} if not self.send_query(qr, gql_token, json_reply): return self.output.test_end(fp) # Unfortunately jmespath is buggy and does not work well for integers :-( # This is unforunate as the hope was to use a jmespath expression # to eliminate all valid sessions (however that might be defined) flatter_json = qr.flatten_json(json_reply, 'router/nodes/flowEntries', '/') if self.debug: print('........ flattened list ..........') pprint.pprint(flatter_json) matching_flows = {} session_flow_counts = {} stats = {} Output.init_result_stats(stats) stats["total_count"] = len(flatter_json) stats["session_flow_count"] = 0 engine = EntryTest.Parser(debug=self.debug) for flow in flatter_json: try: uuid = flow['sessionUuid'] if engine.exclude_entry(flow, exclude_tests): stats["exclude_count"] += 1 continue if not uuid in session_flow_counts: session_flow_counts[uuid] = 1 else: session_flow_counts[uuid] += 1 test_status = engine.eval_entry_by_tests( flow, params["entry_tests"]) Output.update_stats(stats, test_status) if test_status == Output.Status.FAIL: # Note that this must be configured in the parameters just so the value can # be used in this calculation delta = idle_maximum_seconds - flow['inactivityTimeout'] flow["test_idle_duration"] = delta if not uuid in matching_flows or \ matching_flows[uuid]["test_inactivity_duration"] < delta: matching_flows[uuid] = flow except (KeyError, TypeError) as e: flow["test_exception"] = f"Flow Exception: {e}" continue stats["session_flow_count"] = len(session_flow_counts) status = Output.Status.FAIL if len(matching_flows) == 0: status = Output.Status.OK self.output.proc_test_result(status, matching_flows, stats, params) return self.output.test_end(fp)
def run(self, local_info, router_context, gql_token, fp): """ This test uses the gql engine to get device state state Sample data returned by query: [{'name': 'ha-fabric', 'state': None}, {'name': 'DIA', 'state': {'addresses': [{'ipAddress': '172.16.4.103'}]}}, {'name': 'mpls-t1', 'state': {'addresses': [{'ipAddress': '<empty>'}]}}] """ test_info = self.test_info(local_info, router_context) self.output.test_start(test_info) params = self.get_params() include_list = params["network-interfaces"] exclude_list = params["exclude_tests"] # Kind of a hack as we suggest that its OK for an address to be missing # Theoretically this is an error condition, but currently it is normal # that GraphQL will sort of arbitrarily pick a node's L2 HA interfaces # to place state information in. skip_if_address_missing = params["skip_no_address"] """ API = allRouters Fields = name """ qr = gql_helper.NodeGQL("allRouters", ['name'], [ router_context.get_router() ], debug=self.debug) qn = gql_helper.NodeGQL("nodes", ['name']) qd = gql_helper.NodeGQL("deviceInterfaces", [ 'name', 'sharedPhysAddress', 'state { operationalStatus }' ]) qi = gql_helper.NodeGQL("networkInterfaces", ['name', 'state { addresses { ipAddress } }'], include_list) qr.add_node(qn) qn.add_node(qd) qd.add_node(qi) json_reply={} if not self.send_query(qr, gql_token, json_reply): return self.output.test_end(fp) flatter_json = qr.flatten_json(json_reply, 'router/nodes/deviceInterfaces/networkInterfaces', '/') router_context.set_allRouters_node_type(flatter_json) # do not work around the grapqhl api for now... # self._workaround_graphql_api(flatter_json) if self.debug: print('........ flattened list ..........') pprint.pprint(flatter_json) address_list=[] not_excluded_count = 0 if len(flatter_json) > 0: for entry in flatter_json: try: if_name = None if_name = entry['name'] except KeyError: pass if len(include_list) > 0 and \ not if_name in include_list: continue if len(exclude_list) > 0 and \ if_name in exclude_list: continue address=None if if_name is not None: if self.exclude_flat_entry(entry, exclude_list): continue #if entry['router/nodes/deviceInterfaces/state/operationalStatus'] != "OPER_UP" try: address = entry['state']['addresses'][0]['ipAddress'] except (KeyError, IndexError, TypeError) as e: # TODO: Report Exception {e.__class__.__name__} {e} if not skip_if_address_missing: self.output.proc_address_missing(None, entry) continue if address is not None: # address='1.1.1.1' if address == '<empty>': self.output.proc_empty_address(entry) else: ip_address = ipaddress.ip_address(address) status = self.output.status if ip_address.is_private: status = Output.Status.FAIL iptype = 'Private' else: iptype = 'Public' address_list.append(address) self.output.proc_address_type(status, entry, address, iptype) else: if skip_if_address_missing: continue self.output.proc_address_missing(None, entry) not_excluded_count += 1 else: self.output.proc_no_interfaces_found(include_list) status = self.output.status if status == Output.Status.OK and \ not_excluded_count == 0: status = Output.Status.FAIL self.output.proc_test_result(status, address_list, not_excluded_count) return self.output.test_end(fp)
def run(self, local_info, router_context, gql_token, fp): """ This test uses the gql engine to get peer reachability status """ test_info = self.test_info(local_info, router_context) self.output.test_start(test_info) params = self.get_params() # TODO figure out what the include_list is, a list of peers? include_list = params["include_list"] exclusions = params["exclude_tests"] entry_tests = params["entry_tests"] """ API = allNodes Fields = name """ qr = gql_helper.NodeGQL("allRouters", ['name'], [router_context.get_router()], debug=self.debug) qp = gql_helper.NodeGQL("peers", [ 'name', 'paths { node adjacentNode deviceInterface networkInterface adjacentAddress status }' ], include_list) qr.add_node(qp) json_reply = {} if not self.send_query(qr, gql_token, json_reply): return self.output.test_end(fp) # this query is not working correctly unfortunately... even UP is returned flatter_json = qr.flatten_json(json_reply, 'router/peers/paths') router_context.set_allRouters_node_type(flatter_json, 'node') if self.debug: print('........ flattened list ..........') pprint.pprint(flatter_json) paths_per_peer = {} failed_peer_paths = {} stats = {} Output.init_result_stats(stats) stats["total_count"] = len(flatter_json) stats["failed_peer_count"] = 0 stats["tested_peer_count"] = 0 engine = EntryTest.Parser(self.debug) for path in flatter_json: try: if engine.exclude_entry(path, exclusions): stats["exclude_count"] += 1 continue peer_name = path['router/peers/name'] test_result = engine.eval_entry_by_tests(path, entry_tests) Output.update_stats(stats, test_result) if peer_name in paths_per_peer: paths_per_peer[peer_name] += 1 else: paths_per_peer[peer_name] = 1 stats["tested_peer_count"] += 1 if test_result == Output.Status.FAIL: if peer_name in failed_peer_paths: failed_peer_paths[peer_name] += 1 else: failed_peer_paths[peer_name] = 1 stats["failed_peer_count"] += 1 self.output.proc_failed_peer(path, peer_name) self.output.proc_failed_path(path) except KeyError: pass status = Output.Status.OK if stats["FAIL"] > 0: status = Output.Status.FAIL self.output.proc_test_result(entry_tests, stats, status=status) return self.output.test_end(fp)