def test_get_agent_exceptions(self):
neutron_expected = {'neutron-openvswitch-agent':
{'MessagingTimeout': {'2021-03-04': 2},
'AMQP server on 10.10.123.22:5672 is unreachable':
{'2021-03-04': 3},
'OVS is dead': {'2021-03-29': 1},
'RuntimeError': {'2021-03-29': 3}},
'neutron-l3-agent':
{'neutron_lib.exceptions.ProcessExecutionError':
{'2021-05-18': 1}},
}
nova_expected = {'nova-api-wsgi':
{'OSError: Server unexpectedly closed connection':
{'2021-03-15': 1},
'AMQP server on 10.5.1.98:5672 is unreachable':
{'2021-03-15': 1},
'amqp.exceptions.ConnectionForced':
{'2021-03-15': 1}},
'nova-compute':
{'DBConnectionError': {'2021-03-08': 2}}}
barbican_expected = {'barbican-api':
{'UnicodeDecodeError': {'2021-05-04': 1}}}
expected = {"nova": nova_expected, "neutron": neutron_expected,
"barbican": barbican_expected}
s = searchtools.FileSearcher()
c = agent_exceptions.CommonAgentChecks(s)
c.register_search_terms()
results = c.process_results(s.search())
self.assertEqual(results, expected)
def test_filesearcher_network_info(self):
filepath = os.path.join(os.environ["DATA_ROOT"], 'sos_commands',
'networking', 'ip_-d_address')
filepath2 = os.path.join(os.environ["DATA_ROOT"], 'sos_commands',
'networking', 'ip_-s_-d_link')
ip = "10.10.101.33"
mac = "ac:1f:6b:9e:d8:44"
s = searchtools.FileSearcher()
s.add_search_term(r".+({}).+".format(ip), [1], filepath)
s.add_search_term(r"^\s+link/ether\s+({})\s+.+".format(mac), [1],
filepath2)
results = s.search()
self.assertEquals(set(results.files), set([filepath, filepath2]))
self.assertEquals(len(results.find_by_path(filepath)), 1)
self.assertEquals(len(results.find_by_path(filepath2)), 3)
self.assertEquals(results.find_by_path(filepath)[0].linenumber, 16)
for result in results.find_by_path(filepath):
self.assertEquals(result.get(1), ip)
expected = {8: mac,
15: mac,
22: mac}
for result in results.find_by_path(filepath2):
ln = result.linenumber
self.assertEquals(result.tag, None)
self.assertEquals(result.get(1), expected[ln])
def test_process_rpc_loop_results(self):
end = datetime.datetime(2021, 3, 2, 14, 26, 55, 682000)
start = datetime.datetime(2021, 3, 2, 14, 26, 29, 780000)
expected = {
'rpc-loop': {
"top": {
1438: {
'duration': 25.9,
'end': end,
'start': start
}
},
"stats": {
"min": 25.9,
"max": 25.9,
"stdev": 0.0,
"avg": 25.9,
"samples": 1
}
}
}
s = searchtools.FileSearcher()
c = _08agent_checks.NeutronAgentChecks(s)
c.add_rpc_loop_search_terms()
c.process_rpc_loop_results(s.search())
self.assertEqual(c.ovs_agent_info, expected)
def test_get_router_event_stats(self):
router = '9b8efc4c-305b-48ce-a5bd-624bc5eeee67'
spawn_start = datetime.datetime(2021, 3, 25, 18, 10, 14, 747000)
spawn_end = datetime.datetime(2021, 3, 25, 18, 10, 50, 838000)
update_start = datetime.datetime(2021, 3, 25, 18, 9, 54, 720000)
update_end = datetime.datetime(2021, 3, 25, 18, 10, 36, 942000)
expected = {'router-spawn-events': {'stats': {'avg': 36.09,
'max': 36.09,
'min': 36.09,
'samples': 1,
'stdev': 0.0},
'top': {router:
{'duration': 36.091,
'end': spawn_end,
'start': spawn_start}}},
'router-updates': {'stats': {'avg': 28.14,
'max': 42.22,
'min': 14.07,
'samples': 2,
'stdev': 14.08},
'top': {router:
{'duration': 42.222,
'end': update_end,
'start': update_start}}}}
s = searchtools.FileSearcher()
c = agent_checks.NeutronL3AgentEventChecks(s)
c.register_search_terms()
results = c.process_results(s.search())
self.assertEqual(results, expected)
def test_process_rpc_loop_results(self):
end = datetime.datetime(2021, 3, 2, 14, 26, 55, 682000)
start = datetime.datetime(2021, 3, 2, 14, 26, 29, 780000)
expected = {
'rpc-loop': {
"top": {
"1438": {
'duration': 25.9,
'end': end,
'start': start
}
},
"stats": {
"min": 25.9,
"max": 25.9,
"stdev": 0.0,
"avg": 25.9,
"samples": 1
}
}
}
s = searchtools.FileSearcher()
root_key = "neutron-agent-checks"
group_key = "neutron-ovs-agent"
c = neutron_agent_checks.NeutronAgentEventChecks(s, root_key)
c.register_search_terms()
results = c.process_results(s.search())
self.assertEqual(results.get(group_key), expected)
def test_get_agents_bugs(self, mock_add_known_bug):
s = searchtools.FileSearcher()
c = agent_checks.NeutronAgentBugChecks(s)
c.register_search_terms()
results = c.process_results(s.search())
self.assertEqual(results, None)
calls = [mock.call("1896506", ('identified in neutron-l3-agent logs'))]
mock_add_known_bug.assert_has_calls(calls)
def get_agent_exceptions(agent, logs_path, exc_types,
include_time_in_key=False):
"""Search agent logs and determine frequency of occurrences of the given
exception types.
@param agent: (str) name of agent whose logs we want to search.
@param logs_path: (str) path to logs directory
@param exc_types: (list) list of exceptions we want to search for
@param include_time_in_key: (bool) whether to include time of exception in
output. Default is to only show date.
"""
s = searchtools.FileSearcher()
if constants.USE_ALL_LOGS:
data_source = os.path.join(constants.DATA_ROOT, logs_path,
'{}.log*'.format(agent))
else:
data_source = os.path.join(constants.DATA_ROOT, logs_path,
'{}.log'.format(agent))
for exc_type in exc_types:
s.add_search_term(r"^([0-9\-]+) (\S+) .+({}).+".format(exc_type),
[1, 2, 3], data_source)
results = s.search()
agent_exceptions = {}
for path, _results in results:
for result in _results:
exc_tag = result.get(3)
if exc_tag not in agent_exceptions:
agent_exceptions[exc_tag] = {}
if include_time_in_key:
# use hours and minutes only
time = re.compile("([0-9]+:[0-9]+).+").search(result.get(2))[1]
key = "{}_{}".format(result.get(1), time)
else:
key = str(result.get(1))
if key not in agent_exceptions[exc_tag]:
agent_exceptions[exc_tag][key] = 0
agent_exceptions[exc_tag][key] += 1
if not agent_exceptions:
return
for exc_type in agent_exceptions:
agent_exceptions_sorted = {}
for k, v in sorted(agent_exceptions[exc_type].items(),
key=lambda x: x[0]):
agent_exceptions_sorted[k] = v
agent_exceptions[exc_type] = agent_exceptions_sorted
return agent_exceptions
def test_get_agents_issues(self, mock_add_known_bug):
neutron_expected = {
'neutron-openvswitch-agent': {
'MessagingTimeout': {
'2021-03-04': 2
},
'AMQP server on 10.10.123.22:5672 is unreachable': {
'2021-03-04': 3
},
'OVS is dead': {
'2021-03-29': 1
},
'RuntimeError': {
'2021-03-29': 3
},
}
}
nova_expected = {
'nova-api-wsgi': {
'OSError: Server unexpectedly closed connection': {
'2021-03-15': 1
},
'AMQP server on 10.5.1.98:5672 is unreachable': {
'2021-03-15': 1
},
'amqp.exceptions.ConnectionForced': {
'2021-03-15': 1
}
},
'nova-compute': {
'DBConnectionError': {
'2021-03-08': 2
}
}
}
s = searchtools.FileSearcher()
c = _08agent_checks.CommonAgentChecks(s)
c.add_agents_issues_search_terms()
c.process_agent_issues_results(s.search())
self.assertEqual(c.agent_log_issues, {
"neutron": neutron_expected,
"nova": nova_expected
})
calls = [
mock.call("1896506",
description=('identified in neutron-l3-agent logs '
'by testplugin.01part'))
]
mock_add_known_bug.assert_has_calls(calls)
def test_get_agents_issues(self, mock_add_known_bug):
neutron_expected = {'neutron-openvswitch-agent':
{'MessagingTimeout': {'2021-03-04': 2},
'AMQP server on 10.10.123.22:5672 is unreachable':
{'2021-03-04': 3},
'OVS is dead': {'2021-03-29': 1},
'RuntimeError': {'2021-03-29': 3},
}}
nova_expected = {'nova-api-wsgi':
{'OSError: Server unexpectedly closed connection':
{'2021-03-15': 1},
'AMQP server on 10.5.1.98:5672 is unreachable':
{'2021-03-15': 1},
'amqp.exceptions.ConnectionForced':
{'2021-03-15': 1}},
'nova-compute':
{'DBConnectionError': {'2021-03-08': 2}}}
s = searchtools.FileSearcher()
ost_08agent_checks.add_agents_issues_search_terms(s)
ost_08agent_checks.process_agent_issues_results(s.search())
self.assertEqual(ost_08agent_checks.AGENT_LOG_ISSUES,
{"neutron": neutron_expected, "nova": nova_expected})
mock_add_known_bug.assert_has_calls([mock.call("1896506")])
def test_filesearcher_logs(self):
expected = {4: '2021-02-25 14:22:18.861',
16: '2021-02-25 14:22:19.587'}
logs_root = "var/log/neutron/"
filepath = os.path.join(os.environ["DATA_ROOT"], logs_root,
'neutron-openvswitch-agent.log')
globpath = os.path.join(os.environ["DATA_ROOT"], logs_root,
'neutron-l3-agent.log*')
globpath_file1 = os.path.join(os.environ["DATA_ROOT"], logs_root,
'neutron-l3-agent.log')
globpath_file2 = os.path.join(os.environ["DATA_ROOT"], logs_root,
'neutron-l3-agent.log.1.gz')
s = searchtools.FileSearcher()
s.add_search_term(r'^(\S+\s+[0-9:\.]+)\s+.+full sync.+', [1], filepath,
tag="T1")
s.add_search_term(r'^(\S+\s+[0-9:\.]+)\s+.+ERROR.+', [1], filepath,
tag="T2")
s.add_search_term(r'^(\S+\s+[0-9:\.]+)\s+.+INFO.+', [1], globpath,
tag="T3")
# search for something that doesn't exist to test that code path
s.add_search_term(r'non-existant-pattern', [1], globpath,
tag="T4")
results = s.search()
self.assertEquals(set(results.files), set([filepath,
globpath_file2,
globpath_file1]))
self.assertEquals(len(results.find_by_path(filepath)), 6)
tag_results = results.find_by_tag("T1", path=filepath)
self.assertEquals(len(tag_results), 2)
for result in tag_results:
ln = result.linenumber
self.assertEquals(result.tag, "T1")
self.assertEquals(result.get(1), expected[ln])
tag_results = results.find_by_tag("T1")
self.assertEquals(len(tag_results), 2)
for result in tag_results:
ln = result.linenumber
self.assertEquals(result.tag, "T1")
self.assertEquals(result.get(1), expected[ln])
self.assertEquals(len(results.find_by_path(globpath_file1)), 1)
self.assertEquals(len(results.find_by_path(globpath_file2)), 1)
# these files have the same content so expect same result from both
expected = {12: '2021-02-26 14:10:29.729'}
path_results = results.find_by_path(globpath_file1)
for result in path_results:
ln = result.linenumber
self.assertEquals(result.tag, "T3")
self.assertEquals(result.get(1), expected[ln])
path_results = results.find_by_path(globpath_file2)
for result in path_results:
ln = result.linenumber
self.assertEquals(result.tag, "T3")
self.assertEquals(result.get(1), expected[ln])
def test_filesearcher_num_cpus_w_override(self, mock_cpu_count):
mock_cpu_count.return_value = 3
s = searchtools.FileSearcher()
self.assertEquals(s.num_cpus, 2)
