Пример #1
0
    def query_node_info(self):
        """
        If the router name is known, but the nodes are not
        or the nodes are known, but the asset ids are not

        Send a GraphQL query to get the missing information...
        """
        if self.gql_token is None:
            return

        if self.have_asset_info() and \
           self.have_active_info():
            return

        qr = gql_helper.NodeGQL("allRouters", ['name'], [self.get_router()],
                                debug=self.debug)
        qn = gql_helper.NodeGQL("nodes", [
            'name', 'assetId',
            'state { processes { name status primary leaderStatus } }'
        ])

        qr.add_node(qn)

        json_reply = {}
        api_errors = []
        query_status = qr.send_query(self.gql_token, json_reply, api_errors)

        if query_status != 200 or \
            len(api_errors) > 0:
            return

        match_string = "node.state.processes[]"
        merged_list = self._format_allRouters_reply(json_reply, match_string)

        if self.debug:
            print('........ flattened list ..........')
            pprint.pprint(merged_list)

        for entry in merged_list:
            if self.get_asset_by_type(entry['node_type'], load=False) is None:
                # create an asset enty with data available
                asset = {}
                asset['assetId'] = entry['asset_id']
                asset['nodeName'] = entry['node_name']
                asset['t128Version'] = 'Unknown'
                self.update(asset)
            if entry['primary']:
                self.active_proc_map[entry['name']] = entry['node_name']
                self.active_asset_map[entry['name']] = entry['asset_id']
Пример #2
0
  def get_json_assets(self):
      """
      Loads a list of json assets from the conductor
      """
      qn = gql_helper.NodeGQL("allAuthorities", [ 'assets { assetId routerName nodeName t128Version status assetIdConfigured text failedStatus }' ],
                                                    [ ], top_level=True, debug=self.debug)
      json_reply={}
      query_status = qn.send_query(self.gql_token, json_reply)
      if not query_status == 200:
          print('\n*** Unable to query conductor assets ***\n')
          sys.exit(1)

      match_string=f"node.assets[*]"
      flatter_json = jmespath.search(match_string, json_reply)

      if self.debug:
          print('........ flattened list ..........')
          pprint.pprint(flatter_json)

      self.json_assets = flatter_json
      return query_status
Пример #3
0
    def run(self, local_info, router_context, gql_token, fp):
        """
      This test usess the gql engine to get node connectivity status
      """
        test_info = self.test_info(local_info, router_context)
        self.output.test_start(test_info)
        params = self.get_params()
        exclude_list = params["exclude_tests"]
        expected_entries = params["expected_entries"]

        qr = gql_helper.NodeGQL("allRouters", ['name'],
                                [router_context.get_router()],
                                debug=self.debug)
        qn = gql_helper.NodeGQL("nodes", [
            'name',
            'connectivity { status remoteRouterName remoteNodeName remoteNodeRole}'
        ])

        qr.add_node(qn)

        json_reply = {}
        query_status = AbstractTest.GraphQL.ReplyStatus()
        if not self.send_query(qr, gql_token, json_reply, query_status):
            return self.output.test_end(fp)

        flatter_json = qr.flatten_json(json_reply, 'router/nodes/connectivity',
                                       '/')
        router_context.set_allRouters_node_type(flatter_json)

        if self.debug:
            print('........ flattened list ..........')
            pprint.pprint(flatter_json)

        entry_count = 0
        fail_count = 0
        for entry in flatter_json:
            entry_count += 1
            if entry is None:
                # In the past this happens when a node fails to register the 'show
                # system connectivity' command for itself
                self.output.proc_empty_reply(status=Output.Status.FAIL)
                fail_count += 1
                continue
            if self.exclude_flat_entry(entry, exclude_list):
                continue
            self.output.proc_node_status_mismatch(entry)
            fail_count += 1

        expected_entries = params["expected_entries"]
        status = self.output.status
        if query_status.server_code == 200:
            if entry_count < expected_entries:
                self.output.proc_test_result_too_few(entry_count, fail_count,
                                                     params)
            elif status == Output.Status.FAIL:
                self.output.proc_test_result_wrong_state(
                    fail_count, expected_entries, params)
            else:
                self.output.proc_test_result_all_ok(entry_count,
                                                    expected_entries, params)
        else:
            self.output.proc_test_result_bad_query(query_status.server_code)

        return self.output.test_end(fp)
Пример #4
0
  def run(self, local_info, router_context, gql_token, fp):
      """
      This test uses the gql engine to get asset state

      This query only works on the conductor (why? the
      conductor learns asset state from the managed nodes
      so why show it only on the conductor?

      This test will have to include a local node role
      test, which can be checked for 'conductor' and
      return status WARN otherwise...
      """
      test_info = self.test_info(local_info, router_context)
      self.output.test_start(test_info, status=Output.Status.OK)
      params = self.get_params()
      exclude_tests = params["exclude_tests"]
      entry_tests = params["entry_tests"]
      no_match = {}
      if "no_match" in entry_tests:
          no_match = entry_tests["no_match"]

      if local_info.get_node_type() != 'conductor':
          self.output.unsupported_node_type(local_info)
          return self.output.test_end(fp)

      qn = gql_helper.NodeGQL("allAuthorities", [ 'assets { assetId routerName nodeName t128Version status assetIdConfigured ' +
                                                  'text failedStatus }' ], [ ], top_level=True, debug=self.debug)
      json_reply={}
      if not self.send_query(qn, gql_token, json_reply):
          return self.output.test_end(fp)

      #flatter_json = self.flatten_json(json_reply, 'node/assets', '/', prefix='')
      match_string=f"node.assets[*] | [?routerName=='{router_context.get_router()}']"
      flatter_json = jmespath.search(match_string, json_reply)

      router_context.set_allRouters_node_type(flatter_json, node_name_key="nodeName")

      if self.debug:
          print('........ flattened list ..........')
          pprint.pprint(flatter_json)

      stats = {}
      Output.init_result_stats(stats)
      stats["total_count"] = len(flatter_json)

      engine = EntryTest.Parser()
      for entry in flatter_json:
          if engine.exclude_entry(entry, exclude_tests):
              stats["exclude_count"] += 1
              continue
          test_status = engine.eval_entry_by_tests(entry, params["entry_tests"])
          Output.update_stats(stats, test_status)
          if test_status != None:
              self.output.proc_interim_result(entry, status=test_status)

      # All pass versus fail decisions belong outside of the output module
      if stats["FAIL"] > 0:
          self.output.proc_test_result(entry_tests, stats, status=Output.Status.FAIL)
      else:
          self.output.proc_test_result(entry_tests, stats, status=Output.Status.OK)

      return self.output.test_end(fp)
Пример #5
0
  def run(self, local_info, router_context, gql_token, fp):
      fib_table_fields=[
          "totalCount"
      ]
      fib_route_fields=[
          "serviceName " 
          "route { " 
          "ipPrefix "
          "l4Port "
          "l4PortUpper "
          "protocol " 
          "tenant "
          "nextHops { devicePort gateway nodeId vlan deviceInterface networkInterface } "
          "}" 
      ]
      test_info = self.test_info(local_info, router_context)
      self.output.test_start(test_info)
      
      params = self.get_params()
      exclude_tests = params["exclude_tests"]

      fib_entry_suffix=''
      if params["filter_string"] != "" or \
         params["maximum_query_count"] > 0:
          fib_entry_suffix = '('
          if params["maximum_query_count"] > 0:
              fib_entry_suffix += f"first: {params['maximum_query_count']}"
          if params["filter_string"] != "":
              if len(fib_entry_suffix) > 1:
                 fib_entry_suffix += ','
              fib_entry_suffix += f'filter: "\\"\\"~\\"{params["filter_string"]}\\""'
          fib_entry_suffix += ')'

      if local_info.get_router_name() == router_context.get_router() and \
         local_info.get_node_type() == 'conductor':
          self.output.unsupported_node_type(local_info)
          return Output.Status.WARN

      fib_route_data = []
      if len(params["entry_tests"]) > 0:
          fib_route_data = fib_route_fields

      node_names = []
      if 'node_type' in params:
          node_name = router_context.get_node_by_type(params['node_type'])
          if node_name is not None and \
             node_name != '':
              node_names.append(node_name)
      
      # This query is run against all nodes for the router
      qR = gql_helper.NodeGQL("allRouters", ["name"], [router_context.get_router()], debug=self.debug)
      qn = gql_helper.NodeGQL("nodes", ["name"], node_names)
      qf = gql_helper.NodeGQL(f"fibEntries{fib_entry_suffix}", fib_route_data)
      #qr = gql_helper.NodeGQL("route",  fib_route_data)
      qR.add_node(qn)
      qn.add_node(qf)
      #qf.add_node(qr)

      json_reply={}
      if not self.send_query(qR, gql_token, json_reply):
          return self.output.test_end(fp)

      flatter_json = qR.flatten_json(json_reply, 'router/nodes/fibEntries/route', '/')

      if self.debug:
          print('........ flattened list ..........')
          pprint.pprint(flatter_json)

      stats = {}
      Output.init_result_stats(stats)
      stats["total_count"] = len(flatter_json)

      if params["minimum_threshold"] > 0 and \
         total_count < params["minimumm_threshold"]:
          self.output.add_too_few_entries(params)
          return self.output.test_end(fp)

      if params["maximum_threshold"] > 0 and \
         total_count > params["maximum_threshold"]:
          self.output.add_too_man_entries_parameters(params)
          return self.output.test_end(fp)

      engine = EntryTest.Parser(self.debug)
      for entry in flatter_json:
          if engine.exclude_entry(entry, exclude_tests):
              stats["exclude_count"] += 1
              continue
          test_status = engine.eval_entry_by_tests(
              entry, 
              params["entry_tests"]
          )
          if test_status == Output.Status.FAIL:
              test.output.proc_test_match(entry)
              stats["fail_count"] += 1 
          if test_status == Output.Status.WARN:
              test.output.proc_test_match(entry)
              stats["warn_count"] += 1

      self.output.proc_test_result(params, stats)
      return self.output.test_end(fp)
Пример #6
0
    def run(self, local_info, router_context, gql_token, fp):
        """
      This test uses the gql engine to get device state state
      """
        # Process test parameters
        test_info = self.test_info(local_info, router_context)
        self.output.test_start(test_info)
        params = self.get_params()

        # TODO either keep and implement as a parameter or toss
        include_list = params["device-interfaces"]

        # GraophQL Preperation
        qr = gql_helper.NodeGQL("allRouters", ['name'],
                                [router_context.get_router()],
                                debug=self.debug)
        qn = gql_helper.NodeGQL("nodes", ['name'])
        qd = gql_helper.NodeGQL("deviceInterfaces",
                                ['name', 'state { operationalStatus }'],
                                include_list)
        qr.add_node(qn)
        qn.add_node(qd)

        json_reply = {}
        if not self.send_query(qr, gql_token, json_reply):
            return self.output.test_end(fp)

        flatter_json = qr.flatten_json(json_reply,
                                       'router/nodes/deviceInterfaces', '/')

        router_context.set_allRouters_node_type(flatter_json)

        if self.debug:
            print('........ flattened list ..........')
            pprint.pprint(flatter_json)

        # Run test / process results
        fail_count = 0
        warn_count = 0
        engine = EntryTest.Parser()
        for entry in flatter_json:
            try:
                if self.exclude_flat_entry(entry, params["exclude_tests"]):
                    continue
            except KeyError:
                pass

            test_status = engine.eval_entry_by_tests(entry,
                                                     params["entry_tests"])
            if test_status == Output.Status.FAIL:
                self.output.proc_test_match(test_status, entry)
                fail_count += 1

            if test_status == Output.Status.WARN:
                self.output.proc_test_match(test_status, entry)
                warn_count += 1

        if fail_count > 0:
            self.output.proc_test_fail_result(fail_count)
        elif warn_count > 0:
            self.output.proc_test_warn_result(warn_count)

        return self.output.test_end(fp)
Пример #7
0
    def run(self, local_info, router_context, gql_token, fp):
        """
      This test uses the gql engine to learn gateway IP addresses
      and ping them, processing the latency results.

      Note apparently timeout is in seconds and not ms as suggested
      by grapql documentation
      """
        test_info = self.test_info(local_info, router_context)
        self.output.test_start(test_info)
        params = self.get_params()

        # First, query allRouters...nodes...networkInterfaces... to determine
        # the router and gateway
        """
      API    = allRouters
      Fields = name
      """
        intf_list = params["network-interfaces"]
        dev_list = params["device-interfaces"]

        if self.debug:
            print(f'------ Network Interfaces to Process  ------')
            pprint.pprint(intf_list)

        qr = gql_helper.NodeGQL("allRouters", ['name'],
                                [router_context.get_router()],
                                debug=self.debug)
        qn = gql_helper.NodeGQL("nodes", ['name', 'assetId'])
        qd = gql_helper.NodeGQL(
            "deviceInterfaces",
            ['name', 'sharedPhysAddress', 'state { operationalStatus }'],
            dev_list)
        qi = gql_helper.NodeGQL(
            "networkInterfaces",
            ['name', 'state { addresses { ipAddress gateway prefixLength } }'],
            intf_list)
        qa = gql_helper.NodeGQL("addresses",
                                ['ipAddress', 'gateway', 'prefixLength'])
        qr.add_node(qn)
        qn.add_node(qd)
        qd.add_node(qi)
        qi.add_node(qa)

        json_reply = {}
        if not self.send_query(qr, gql_token, json_reply):
            return self.output.test_end(fp)

        if params["static-address"]:
            flatter_json = qr.flatten_json(
                json_reply,
                'router/nodes/deviceInterfaces/networkInterfaces/addresses',
                '/')
            ni_name_key = 'router/nodes/deviceInterfaces/networkInterfaces/name'
        else:
            # stop prefix should be router/nodes/deviceInterfaces/networkInterfaces/state, but because no address is
            # returned for one of the peers (state=None), flatten_json skips that node. So go one level higher and
            # live with it for now.
            flatter_json = qr.flatten_json(
                json_reply, 'router/nodes/deviceInterfaces/networkInterfaces',
                '/')
            ni_name_key = 'name'

        router_context.set_allRouters_node_type(flatter_json)

        self._workaround_graphql_api(flatter_json)

        if self.debug:
            print('........ flattened list ..........')
            pprint.pprint(flatter_json)

        self.output.progress_start(fp)

        gateway_success_count = 0
        gateway_fail_count = 0
        gateway_count = 0

        for entry in flatter_json:
            if self.debug:
                print(f'%%% process NI for Ping %%%')
                pprint.pprint(entry)
            if params["node_type"] != '' and \
                entry['node_type'] != params["node_type"]:
                if self.debug:
                    print(
                        f"NI {entry[ni_name_key]}: Skipping {entry['node_type']} node {entry['node_name']}"
                    )
                continue
            address = ''
            gateway = ''
            try:
                if params["static-address"]:
                    addresses = [entry]
                else:
                    addresses = entry['state']['addresses']
                if self.debug:
                    print(f'%%%% process address for Ping %%%%')
                    pprint.pprint(addresses)
                egress_interface = entry[ni_name_key]
                for address_entry in addresses:
                    address = address_entry['ipAddress']
                    gateway = address_entry['gateway']
                    prefix_length = int(address_entry['prefixLength'])
                    target = gateway
                    dest_ip = None

                    if gateway is None:
                        # hack for ipv4 only!
                        hack = int(ipaddress.IPv4Address(address))
                        if prefix_length == 31:
                            gateway_hack = hack & 0xfffffffe | -(hack
                                                                 & 0x01) + 1
                            gateway_ip = ipaddress.IPv4Address(gateway_hack)
                            dest_ip = str(gateway_ip)
                            gateway = ''
                            target = dest_ip
                        elif prefix_length == 30:
                            gateway_hack = hack & 0xfffffffc | -(hack
                                                                 & 0x03) + 3
                            gateway_ip = ipaddress.IPv4Address(gateway_hack)
                            dest_ip = str(gateway_ip)
                            gateway = ''
                            target = dest_ip
                        else:
                            self.output.proc_cannot_ping_no_gateway(
                                entry, ni_name_key)
                            msg_list.append(
                                f'Cannot ping via NI {entry[ni_name_key]}, No Gateway!'
                            )
                            gateway_count += 1
                            continue

                    try:
                        oper_status = entry[
                            'router/nodes/deviceInterfaces/state/operationalStatus']
                        if oper_status != 'OPER_UP':
                            self.output.proc_cannot_ping_dev_status(
                                entry, ni_name_key, oper_status)
                            gateway_count += 1
                            #continue
                            break
                    except KeyError:
                        # Continue as there is no operationalStatus to evaluate
                        pass

                    # Invoke Graphql PING API
                    ping_count = 0
                    if dest_ip is None:
                        if "destination-ip" in params and \
                            params["destination-ip"] != '':
                            dest_ip = params["destination-ip"]
                        else:
                            dest_ip = gateway
                    size = params["size"]
                    timeout = params["timeout"]
                    seqno = params["sequence"]
                    router = entry["router/name"]
                    node = entry["node_name"]
                    identifier = params["identifier"]

                    total_response_time = float(0)
                    average_response_time = float(0)
                    ping_success_count = 0
                    ping_fail_count = 0

                    while ping_count < params["iterations"]:
                        argstr = f'routerName: "{router}"'
                        argstr += f', nodeName: "{node}"'
                        argstr += f', identifier: {identifier}'
                        argstr += f', egressInterface: "{egress_interface}"'
                        if dest_ip != '':
                            argstr += f', destinationIp: "{dest_ip}"'
                        if gateway != '':
                            argstr += f', gatewayIp: "{gateway}"'
                        argstr += f', sequence: {seqno}'
                        if size != '':
                            argstr += f', size: {size}'
                        argstr += f', timeout: {timeout}'
                        if self.debug:
                            print(f'argstr={argstr}')

                        # display progress in-place as does 128status.sh...
                        now_message = f"NI {entry[ni_name_key]}: ping {gateway} {ping_count}/{params['iterations']} tmo={params['timeout']}s"
                        self.output.progress_display(now_message, fp)
                        qp = gql_helper.RawGQL(
                            f'ping({argstr}) ' +
                            '{ status statusReason reachable sequence ttl responseTime }',
                            debug=self.debug)
                        json_ping_reply = {}
                        qp.send_query(gql_token, json_ping_reply)

                        # standard graphql error processing may not be appropriate here as a failure can
                        # be part of the test process w/o ruining the test.

                        ping_count += 1
                        seqno += 1

                        try:
                            # "0" used < 4.2.0; "SUCCESS" used in 4.2.0+
                            json_ping_reply = json_ping_reply['ping']
                            if json_ping_reply['status'] == "0" or \
                               json_ping_reply['status'] == "SUCCESS":
                                ping_success_count += 1
                                total_response_time += float(
                                    json_ping_reply['responseTime'])
                                average_response_time = total_response_time / float(
                                    ping_success_count)
                            else:
                                ping_fail_count += 1
                        except (KeyError, TypeError) as e:
                            self.output.proc_no_data_in_reply(
                                entry, ni_name_key, gateway)
                            ping_fail_count += 1
                            gateway_count += 1
                            continue

                    if ping_count == ping_success_count:
                        # fix this for multiple matching entries
                        gateway_success_count += 1
                        self.output.proc_ping_result_pass(
                            entry, ni_name_key, ping_count, ping_success_count,
                            target, average_response_time)
                    else:
                        gateway_fail_count += 1
                        self.output.proc_ping_result_fail(
                            entry, ni_name_key, ping_count, ping_fail_count,
                            target, average_response_time)
                gateway_count += 1

            except (TypeError) as e:
                self.output.proc_no_address_in_reply(entry, ni_name_key)
                continue

        status = self.output.status
        if gateway_count == 0:
            status = Output.Status.WARN
        if gateway_count != gateway_success_count:
            status = Output.Status.FAIL

        self.output.proc_test_result(status, gateway_count,
                                     gateway_success_count, gateway_fail_count,
                                     params)
        return self.output.test_end(fp)
Пример #8
0
    def run(self, local_info, router_context, gql_token, fp):
        flowEntryFields = [ \
            'sourceIp',
            'destIp',
            'sourcePort',
            'destPort',
            'vlan',
            'devicePort',
            'protocol',
            'sessionUuid',
            'natIp',
            'natPort',
            'serviceName',
            'tenant',
            'encrypted',
            'inactivityTimeout',
            'deviceInterfaceName',
            'networkInterfaceName',
            'startTime',
            'forward'
          ]

        test_info = self.test_info(local_info, router_context)
        self.output.test_start(test_info)

        params = self.get_params()
        try:
            idle_threshold_seconds = params["idle_threshold_seconds"]
            idle_maximum_seconds = params["idle_maximum_seconds"]
            max_sessions = params["max_sessions_to_query"]
            filter_string = params["filter_string"]
            match_port = params["match_port"]
        except Exception as e:
            # TODO: Improve error handling
            print("CONFIG ERROR\n")
            return Output.Status.FAIL

        exclude_tests = []
        if "exclude_tests" in params:
            exclude_tests = params["exclude_tests"]

        flow_entry_suffix = f'(first: {max_sessions}, filter: "\\"\\"~\\"{filter_string}\\"")'

        if local_info.get_router_name() == router_context.get_router() and \
           local_info.get_node_type() == 'conductor':
            # Check Error output
            self.output.unsupported_node_type(local_info)
            return Output.Status.WARN

        qr = gql_helper.NodeGQL("allRouters", ['name'],
                                [router_context.get_router()],
                                debug=self.debug)
        qn = gql_helper.NodeGQL("nodes", ['name'])
        qf = gql_helper.NodeGQL(f"flowEntries{flow_entry_suffix}",
                                flowEntryFields)
        qr.add_node(qn)
        qn.add_node(qf)

        json_reply = {}
        if not self.send_query(qr, gql_token, json_reply):
            return self.output.test_end(fp)

        # Unfortunately jmespath is buggy and does not work well for integers :-(
        # This is unforunate as the hope was to use a jmespath expression
        # to eliminate all valid sessions (however that might be defined)
        flatter_json = qr.flatten_json(json_reply, 'router/nodes/flowEntries',
                                       '/')

        if self.debug:
            print('........ flattened list ..........')
            pprint.pprint(flatter_json)

        matching_flows = {}
        session_flow_counts = {}
        stats = {}
        Output.init_result_stats(stats)
        stats["total_count"] = len(flatter_json)
        stats["session_flow_count"] = 0

        engine = EntryTest.Parser(debug=self.debug)
        for flow in flatter_json:
            try:
                uuid = flow['sessionUuid']
                if engine.exclude_entry(flow, exclude_tests):
                    stats["exclude_count"] += 1
                    continue
                if not uuid in session_flow_counts:
                    session_flow_counts[uuid] = 1
                else:
                    session_flow_counts[uuid] += 1
                test_status = engine.eval_entry_by_tests(
                    flow, params["entry_tests"])
                Output.update_stats(stats, test_status)
                if test_status == Output.Status.FAIL:
                    # Note that this must be configured in the parameters just so the value can
                    # be used in this calculation
                    delta = idle_maximum_seconds - flow['inactivityTimeout']
                    flow["test_idle_duration"] = delta
                    if not uuid in matching_flows or \
                       matching_flows[uuid]["test_inactivity_duration"] < delta:
                        matching_flows[uuid] = flow

            except (KeyError, TypeError) as e:
                flow["test_exception"] = f"Flow Exception: {e}"
                continue

        stats["session_flow_count"] = len(session_flow_counts)

        status = Output.Status.FAIL
        if len(matching_flows) == 0:
            status = Output.Status.OK

        self.output.proc_test_result(status, matching_flows, stats, params)
        return self.output.test_end(fp)
Пример #9
0
  def run(self, local_info, router_context, gql_token, fp):
      """
      This test uses the gql engine to get device state state

      Sample data returned by query:
      [{'name': 'ha-fabric', 'state': None},
       {'name': 'DIA', 'state': {'addresses': [{'ipAddress': '172.16.4.103'}]}},
       {'name': 'mpls-t1', 'state': {'addresses': [{'ipAddress': '<empty>'}]}}]
      """
      test_info = self.test_info(local_info, router_context)
      self.output.test_start(test_info)
      params = self.get_params()
      include_list = params["network-interfaces"]
      exclude_list = params["exclude_tests"]
      
      # Kind of a hack as we suggest that its OK for an address to be missing
      # Theoretically this is an error condition, but currently it is normal
      # that GraphQL will sort of arbitrarily pick a node's L2 HA interfaces
      # to place state information in.
      skip_if_address_missing = params["skip_no_address"]

      """
      API    = allRouters
      Fields = name
      """
      qr = gql_helper.NodeGQL("allRouters", ['name'], [ router_context.get_router() ], debug=self.debug)
      qn = gql_helper.NodeGQL("nodes", ['name'])
      qd = gql_helper.NodeGQL("deviceInterfaces", [ 'name', 'sharedPhysAddress', 'state { operationalStatus }' ])
      qi = gql_helper.NodeGQL("networkInterfaces", ['name', 'state { addresses { ipAddress } }'], include_list)
      qr.add_node(qn)
      qn.add_node(qd)
      qd.add_node(qi)

      json_reply={}
      if not self.send_query(qr, gql_token, json_reply):
          return self.output.test_end(fp)

      flatter_json = qr.flatten_json(json_reply, 'router/nodes/deviceInterfaces/networkInterfaces', '/')

      router_context.set_allRouters_node_type(flatter_json)
 
      # do not work around the grapqhl api for now...
      # self._workaround_graphql_api(flatter_json)

      if self.debug:
          print('........ flattened list ..........')
          pprint.pprint(flatter_json)

      address_list=[]
      not_excluded_count = 0
      if len(flatter_json) > 0:
          for entry in flatter_json:
              try:
                  if_name = None
                  if_name = entry['name']
              except KeyError:
                  pass
              if len(include_list) > 0 and \
                  not if_name in include_list:
                  continue
              if len(exclude_list) > 0 and \
                  if_name in exclude_list:
                  continue
              address=None
              if if_name is not None:
                  if self.exclude_flat_entry(entry, exclude_list):
                      continue
                  #if entry['router/nodes/deviceInterfaces/state/operationalStatus'] != "OPER_UP"
                  try:
                      address = entry['state']['addresses'][0]['ipAddress']
                  except (KeyError, IndexError, TypeError) as e:
                      # TODO: Report Exception {e.__class__.__name__} {e}
                      if not skip_if_address_missing:
                          self.output.proc_address_missing(None, entry)
                      continue
                  if address is not None:
                      #  address='1.1.1.1'
                      if address == '<empty>':
                          self.output.proc_empty_address(entry)
                      else:  
                          ip_address = ipaddress.ip_address(address)
                          status = self.output.status
                          if ip_address.is_private:
                              status = Output.Status.FAIL
                              iptype = 'Private'
                          else:
                              iptype = 'Public'
                              address_list.append(address)
                          self.output.proc_address_type(status, entry, address, iptype)
                  else:
                      if skip_if_address_missing:
                         continue
                      self.output.proc_address_missing(None, entry)
                  not_excluded_count += 1
      else:
          self.output.proc_no_interfaces_found(include_list)

      status = self.output.status
      if status == Output.Status.OK and \
         not_excluded_count == 0:
          status = Output.Status.FAIL

      self.output.proc_test_result(status, address_list, not_excluded_count)
      return self.output.test_end(fp)
    def run(self, local_info, router_context, gql_token, fp):
        """
      This test uses the gql engine to get peer reachability status
      """
        test_info = self.test_info(local_info, router_context)
        self.output.test_start(test_info)
        params = self.get_params()
        # TODO figure out what the include_list is, a list of peers?
        include_list = params["include_list"]
        exclusions = params["exclude_tests"]
        entry_tests = params["entry_tests"]
        """
      API    = allNodes
      Fields = name
      """
        qr = gql_helper.NodeGQL("allRouters", ['name'],
                                [router_context.get_router()],
                                debug=self.debug)
        qp = gql_helper.NodeGQL("peers", [
            'name',
            'paths { node adjacentNode deviceInterface networkInterface adjacentAddress status }'
        ], include_list)
        qr.add_node(qp)

        json_reply = {}
        if not self.send_query(qr, gql_token, json_reply):
            return self.output.test_end(fp)

        # this query is not working correctly unfortunately... even UP is returned
        flatter_json = qr.flatten_json(json_reply, 'router/peers/paths')

        router_context.set_allRouters_node_type(flatter_json, 'node')

        if self.debug:
            print('........ flattened list ..........')
            pprint.pprint(flatter_json)

        paths_per_peer = {}
        failed_peer_paths = {}
        stats = {}
        Output.init_result_stats(stats)
        stats["total_count"] = len(flatter_json)
        stats["failed_peer_count"] = 0
        stats["tested_peer_count"] = 0

        engine = EntryTest.Parser(self.debug)
        for path in flatter_json:
            try:
                if engine.exclude_entry(path, exclusions):
                    stats["exclude_count"] += 1
                    continue
                peer_name = path['router/peers/name']
                test_result = engine.eval_entry_by_tests(path, entry_tests)
                Output.update_stats(stats, test_result)
                if peer_name in paths_per_peer:
                    paths_per_peer[peer_name] += 1
                else:
                    paths_per_peer[peer_name] = 1
                    stats["tested_peer_count"] += 1
                if test_result == Output.Status.FAIL:
                    if peer_name in failed_peer_paths:
                        failed_peer_paths[peer_name] += 1
                    else:
                        failed_peer_paths[peer_name] = 1
                        stats["failed_peer_count"] += 1
                        self.output.proc_failed_peer(path, peer_name)
                    self.output.proc_failed_path(path)
            except KeyError:
                pass

        status = Output.Status.OK
        if stats["FAIL"] > 0:
            status = Output.Status.FAIL

        self.output.proc_test_result(entry_tests, stats, status=status)
        return self.output.test_end(fp)