def test_get_agent_exceptions(self): neutron_expected = {'neutron-openvswitch-agent': {'MessagingTimeout': {'2021-03-04': 2}, 'AMQP server on 10.10.123.22:5672 is unreachable': {'2021-03-04': 3}, 'OVS is dead': {'2021-03-29': 1}, 'RuntimeError': {'2021-03-29': 3}}, 'neutron-l3-agent': {'neutron_lib.exceptions.ProcessExecutionError': {'2021-05-18': 1}}, } nova_expected = {'nova-api-wsgi': {'OSError: Server unexpectedly closed connection': {'2021-03-15': 1}, 'AMQP server on 10.5.1.98:5672 is unreachable': {'2021-03-15': 1}, 'amqp.exceptions.ConnectionForced': {'2021-03-15': 1}}, 'nova-compute': {'DBConnectionError': {'2021-03-08': 2}}} barbican_expected = {'barbican-api': {'UnicodeDecodeError': {'2021-05-04': 1}}} expected = {"nova": nova_expected, "neutron": neutron_expected, "barbican": barbican_expected} s = searchtools.FileSearcher() c = agent_exceptions.CommonAgentChecks(s) c.register_search_terms() results = c.process_results(s.search()) self.assertEqual(results, expected)
def test_filesearcher_network_info(self): filepath = os.path.join(os.environ["DATA_ROOT"], 'sos_commands', 'networking', 'ip_-d_address') filepath2 = os.path.join(os.environ["DATA_ROOT"], 'sos_commands', 'networking', 'ip_-s_-d_link') ip = "10.10.101.33" mac = "ac:1f:6b:9e:d8:44" s = searchtools.FileSearcher() s.add_search_term(r".+({}).+".format(ip), [1], filepath) s.add_search_term(r"^\s+link/ether\s+({})\s+.+".format(mac), [1], filepath2) results = s.search() self.assertEquals(set(results.files), set([filepath, filepath2])) self.assertEquals(len(results.find_by_path(filepath)), 1) self.assertEquals(len(results.find_by_path(filepath2)), 3) self.assertEquals(results.find_by_path(filepath)[0].linenumber, 16) for result in results.find_by_path(filepath): self.assertEquals(result.get(1), ip) expected = {8: mac, 15: mac, 22: mac} for result in results.find_by_path(filepath2): ln = result.linenumber self.assertEquals(result.tag, None) self.assertEquals(result.get(1), expected[ln])
def test_process_rpc_loop_results(self): end = datetime.datetime(2021, 3, 2, 14, 26, 55, 682000) start = datetime.datetime(2021, 3, 2, 14, 26, 29, 780000) expected = { 'rpc-loop': { "top": { 1438: { 'duration': 25.9, 'end': end, 'start': start } }, "stats": { "min": 25.9, "max": 25.9, "stdev": 0.0, "avg": 25.9, "samples": 1 } } } s = searchtools.FileSearcher() c = _08agent_checks.NeutronAgentChecks(s) c.add_rpc_loop_search_terms() c.process_rpc_loop_results(s.search()) self.assertEqual(c.ovs_agent_info, expected)
def test_get_router_event_stats(self): router = '9b8efc4c-305b-48ce-a5bd-624bc5eeee67' spawn_start = datetime.datetime(2021, 3, 25, 18, 10, 14, 747000) spawn_end = datetime.datetime(2021, 3, 25, 18, 10, 50, 838000) update_start = datetime.datetime(2021, 3, 25, 18, 9, 54, 720000) update_end = datetime.datetime(2021, 3, 25, 18, 10, 36, 942000) expected = {'router-spawn-events': {'stats': {'avg': 36.09, 'max': 36.09, 'min': 36.09, 'samples': 1, 'stdev': 0.0}, 'top': {router: {'duration': 36.091, 'end': spawn_end, 'start': spawn_start}}}, 'router-updates': {'stats': {'avg': 28.14, 'max': 42.22, 'min': 14.07, 'samples': 2, 'stdev': 14.08}, 'top': {router: {'duration': 42.222, 'end': update_end, 'start': update_start}}}} s = searchtools.FileSearcher() c = agent_checks.NeutronL3AgentEventChecks(s) c.register_search_terms() results = c.process_results(s.search()) self.assertEqual(results, expected)
def test_process_rpc_loop_results(self): end = datetime.datetime(2021, 3, 2, 14, 26, 55, 682000) start = datetime.datetime(2021, 3, 2, 14, 26, 29, 780000) expected = { 'rpc-loop': { "top": { "1438": { 'duration': 25.9, 'end': end, 'start': start } }, "stats": { "min": 25.9, "max": 25.9, "stdev": 0.0, "avg": 25.9, "samples": 1 } } } s = searchtools.FileSearcher() root_key = "neutron-agent-checks" group_key = "neutron-ovs-agent" c = neutron_agent_checks.NeutronAgentEventChecks(s, root_key) c.register_search_terms() results = c.process_results(s.search()) self.assertEqual(results.get(group_key), expected)
def test_get_agents_bugs(self, mock_add_known_bug): s = searchtools.FileSearcher() c = agent_checks.NeutronAgentBugChecks(s) c.register_search_terms() results = c.process_results(s.search()) self.assertEqual(results, None) calls = [mock.call("1896506", ('identified in neutron-l3-agent logs'))] mock_add_known_bug.assert_has_calls(calls)
def get_agent_exceptions(agent, logs_path, exc_types, include_time_in_key=False): """Search agent logs and determine frequency of occurrences of the given exception types. @param agent: (str) name of agent whose logs we want to search. @param logs_path: (str) path to logs directory @param exc_types: (list) list of exceptions we want to search for @param include_time_in_key: (bool) whether to include time of exception in output. Default is to only show date. """ s = searchtools.FileSearcher() if constants.USE_ALL_LOGS: data_source = os.path.join(constants.DATA_ROOT, logs_path, '{}.log*'.format(agent)) else: data_source = os.path.join(constants.DATA_ROOT, logs_path, '{}.log'.format(agent)) for exc_type in exc_types: s.add_search_term(r"^([0-9\-]+) (\S+) .+({}).+".format(exc_type), [1, 2, 3], data_source) results = s.search() agent_exceptions = {} for path, _results in results: for result in _results: exc_tag = result.get(3) if exc_tag not in agent_exceptions: agent_exceptions[exc_tag] = {} if include_time_in_key: # use hours and minutes only time = re.compile("([0-9]+:[0-9]+).+").search(result.get(2))[1] key = "{}_{}".format(result.get(1), time) else: key = str(result.get(1)) if key not in agent_exceptions[exc_tag]: agent_exceptions[exc_tag][key] = 0 agent_exceptions[exc_tag][key] += 1 if not agent_exceptions: return for exc_type in agent_exceptions: agent_exceptions_sorted = {} for k, v in sorted(agent_exceptions[exc_type].items(), key=lambda x: x[0]): agent_exceptions_sorted[k] = v agent_exceptions[exc_type] = agent_exceptions_sorted return agent_exceptions
def test_get_agents_issues(self, mock_add_known_bug): neutron_expected = { 'neutron-openvswitch-agent': { 'MessagingTimeout': { '2021-03-04': 2 }, 'AMQP server on 10.10.123.22:5672 is unreachable': { '2021-03-04': 3 }, 'OVS is dead': { '2021-03-29': 1 }, 'RuntimeError': { '2021-03-29': 3 }, } } nova_expected = { 'nova-api-wsgi': { 'OSError: Server unexpectedly closed connection': { '2021-03-15': 1 }, 'AMQP server on 10.5.1.98:5672 is unreachable': { '2021-03-15': 1 }, 'amqp.exceptions.ConnectionForced': { '2021-03-15': 1 } }, 'nova-compute': { 'DBConnectionError': { '2021-03-08': 2 } } } s = searchtools.FileSearcher() c = _08agent_checks.CommonAgentChecks(s) c.add_agents_issues_search_terms() c.process_agent_issues_results(s.search()) self.assertEqual(c.agent_log_issues, { "neutron": neutron_expected, "nova": nova_expected }) calls = [ mock.call("1896506", description=('identified in neutron-l3-agent logs ' 'by testplugin.01part')) ] mock_add_known_bug.assert_has_calls(calls)
def test_get_agents_issues(self, mock_add_known_bug): neutron_expected = {'neutron-openvswitch-agent': {'MessagingTimeout': {'2021-03-04': 2}, 'AMQP server on 10.10.123.22:5672 is unreachable': {'2021-03-04': 3}, 'OVS is dead': {'2021-03-29': 1}, 'RuntimeError': {'2021-03-29': 3}, }} nova_expected = {'nova-api-wsgi': {'OSError: Server unexpectedly closed connection': {'2021-03-15': 1}, 'AMQP server on 10.5.1.98:5672 is unreachable': {'2021-03-15': 1}, 'amqp.exceptions.ConnectionForced': {'2021-03-15': 1}}, 'nova-compute': {'DBConnectionError': {'2021-03-08': 2}}} s = searchtools.FileSearcher() ost_08agent_checks.add_agents_issues_search_terms(s) ost_08agent_checks.process_agent_issues_results(s.search()) self.assertEqual(ost_08agent_checks.AGENT_LOG_ISSUES, {"neutron": neutron_expected, "nova": nova_expected}) mock_add_known_bug.assert_has_calls([mock.call("1896506")])
def test_filesearcher_logs(self): expected = {4: '2021-02-25 14:22:18.861', 16: '2021-02-25 14:22:19.587'} logs_root = "var/log/neutron/" filepath = os.path.join(os.environ["DATA_ROOT"], logs_root, 'neutron-openvswitch-agent.log') globpath = os.path.join(os.environ["DATA_ROOT"], logs_root, 'neutron-l3-agent.log*') globpath_file1 = os.path.join(os.environ["DATA_ROOT"], logs_root, 'neutron-l3-agent.log') globpath_file2 = os.path.join(os.environ["DATA_ROOT"], logs_root, 'neutron-l3-agent.log.1.gz') s = searchtools.FileSearcher() s.add_search_term(r'^(\S+\s+[0-9:\.]+)\s+.+full sync.+', [1], filepath, tag="T1") s.add_search_term(r'^(\S+\s+[0-9:\.]+)\s+.+ERROR.+', [1], filepath, tag="T2") s.add_search_term(r'^(\S+\s+[0-9:\.]+)\s+.+INFO.+', [1], globpath, tag="T3") # search for something that doesn't exist to test that code path s.add_search_term(r'non-existant-pattern', [1], globpath, tag="T4") results = s.search() self.assertEquals(set(results.files), set([filepath, globpath_file2, globpath_file1])) self.assertEquals(len(results.find_by_path(filepath)), 6) tag_results = results.find_by_tag("T1", path=filepath) self.assertEquals(len(tag_results), 2) for result in tag_results: ln = result.linenumber self.assertEquals(result.tag, "T1") self.assertEquals(result.get(1), expected[ln]) tag_results = results.find_by_tag("T1") self.assertEquals(len(tag_results), 2) for result in tag_results: ln = result.linenumber self.assertEquals(result.tag, "T1") self.assertEquals(result.get(1), expected[ln]) self.assertEquals(len(results.find_by_path(globpath_file1)), 1) self.assertEquals(len(results.find_by_path(globpath_file2)), 1) # these files have the same content so expect same result from both expected = {12: '2021-02-26 14:10:29.729'} path_results = results.find_by_path(globpath_file1) for result in path_results: ln = result.linenumber self.assertEquals(result.tag, "T3") self.assertEquals(result.get(1), expected[ln]) path_results = results.find_by_path(globpath_file2) for result in path_results: ln = result.linenumber self.assertEquals(result.tag, "T3") self.assertEquals(result.get(1), expected[ln])
def test_filesearcher_num_cpus_w_override(self, mock_cpu_count): mock_cpu_count.return_value = 3 s = searchtools.FileSearcher() self.assertEquals(s.num_cpus, 2)