def test_generate_events_normal(self):
"""
Test _generate_events under normal operation, using all net namespaces
"""
tracer = object.__new__(ebpf.TCPTracer)
tracer.options = None
data = StringIO(
"header1\n"
"time type pid comm ip s_addr d_addr s_port d_port size netns\n"
"1 A 2 comm1 4 127. 127. 3 4 1 5\n"
"6 B 7 comm2 4 127. 127. 4 3 1 5\n"
"1 A 2 comm3 4 x x 3 4 1 5\n"
)
port_dict = {4: {('pid1', 'test1')}, 3: {('pid2', 'test2')}}
result = list(tracer._generate_events(data, port_dict))
expected = [
data_io.EventDatum(time=1,
type='A',
connected=[('source_', 'dest_')],
specific_datum={
"source_pid": 2,
"source_comm": 'comm1',
"source_port": 3,
"dest_pid": 'pid1',
"dest_comm": 'test1',
"dest_port": 4,
"size": 1,
"net_ns": 5
}),
data_io.EventDatum(time=6,
type='B',
connected=[('source_', 'dest_')],
specific_datum={
"source_pid": 7,
"source_comm": 'comm2',
"source_port": 4,
"dest_pid": 'pid2',
"dest_comm": 'test2',
"dest_port": 3,
"size": 1,
"net_ns": 5
})
]
self.assertEqual(expected, result)
async def test_success(self, release_mock, re_mock):
""" Test successful regex matching. """
# Set up mocks
release_mock.return_value = "100.0.0" # so we ignore the kernel check
self.strio_mock.return_value = StringIO('test_out2')
match_mock = re_mock.match.return_value
match_mock.group.side_effect = [
"111.999", "test_pid", "test_name", "4", "test_event"
]
collecter = perf.SchedulingEvents(self.time)
data = await collecter.collect()
sched_events = list(data.datum_generator)
self.create_mock.assert_has_calls([
asynctest.call("perf sched record -o " +
perf.SchedulingEvents._PERF_FILE_NAME + " sleep " +
str(self.time),
stderr=self.pipe_mock),
asynctest.call().communicate(),
asynctest.call("perf sched script -i " +
perf.SchedulingEvents._PERF_FILE_NAME +
" -F 'comm,pid,cpu,time,event'",
stdout=self.pipe_mock,
stderr=self.pipe_mock),
asynctest.call().communicate()
])
# self.log_mock.error.assert_has_calls([
# asynctest.call("test_err1"),
# asynctest.call("test_err2")
# ])
self.os_mock.remove.assert_called_once_with(
self.os_mock.getcwd.return_value + "/" +
perf.SchedulingEvents._PERF_FILE_NAME)
re_mock.match.assert_called_once_with(
r"\s*"
r"(?P<name>\S+(\s+\S+)*)\s+"
r"(?P<pid>\d+)\s+"
r"\[(?P<cpu>\d+)\]\s+"
r"(?P<time>\d+.\d+):\s+"
r"(?P<event>\S+)", "test_out2")
expected_event = data_io.EventDatum(
specific_datum={
'pid': 'test_pid',
'cpu': '4',
'comm': 'test_name'
},
#"test_name (pid: test_pid)", "cpu 4"),
time=111000999,
type="test_event",
connected=None)
self.assertEqual([expected_event], sched_events)
def test_with_newline_rn(self):
"""Ensure from_string copes with newlines"""
expected = data_io.EventDatum(
time=1, type="type",
specific_datum={'pid': 'p1', 'comm': 'n1', 'cpu': 'c1'},
connected=[('source_', 'dest_')])
x = "1" + consts.field_separator + "type" + consts.field_separator + \
"{'pid': 'p1', 'comm': 'n1', 'cpu': 'c1'}" + \
consts.field_separator + "[('source_', 'dest_')]"
self.check_from_str(x + '\r\n', expected)
def test_to_string(self):
"""Test datapoints are correctly converted to strings."""
se = data_io.EventDatum(
time=1, type="type",
specific_datum={'pid': 'p1', 'comm': 'n1', 'cpu': 'c1'},
connected=[('source_', 'dest_')])
expected = "1" + consts.field_separator + "type" + consts.field_separator + \
"{'pid': 'p1', 'comm': 'n1', 'cpu': 'c1'}" + \
consts.field_separator + "[('source_', 'dest_')]"
actual = str(se)
self.assertEqual(expected, actual, msg='Expected {}, got {}'
.format(expected, actual))
def _normalise(self):
"""
Normalises the times in the events so they start at 0
Method that subtracts the minimum time from all the event times so that
they start at 0. Since the EventDatum is imutable we can't just change
the time, so we create new events with the new times.
"""
for partition in self.event_partition.values():
for idx, event in enumerate(partition):
partition[idx] = data_io.EventDatum(event.time - self.min_time,
event.type,
event.specific_datum,
event.connected)
def test_generate_events_full_dict(self, output_mock):
"""
Test _generate_events under normal operation, using all net namespaces
"""
tracer = object.__new__(ebpf.TCPTracer)
tracer.options = None
data = StringIO(
"header1\n"
"time type pid comm ip s_addr d_addr s_port d_port size netns\n"
"1 A 2 comm1 4 127. 127. 3 4 1 5\n"
"6 B 7 comm2 4 127. 127. 4 3 1 5\n"
"1 A 2 comm3 4 x x 3 4 1 5\n"
)
port_dict = {
4: {('pid1', 'test1')},
3: {('pid2', 'test2'), ('pid3', 'test3')}
}
result = list(tracer._generate_events(data, port_dict))
expected = [
data_io.EventDatum(time=1,
type='A',
connected=[('source_', 'dest_')],
specific_datum={
"source_pid": 2,
"source_comm": 'comm1',
"source_port": 3,
"dest_pid": 'pid1',
"dest_comm": 'test1',
"dest_port": 4,
"size": 1,
"net_ns": 5
})
]
self.assertEqual(expected, result)
expected_errors = [
asynctest.call(
text="IPC: Too many destination port PIDs/comms found. "
"Check log for details.",
description="Too many destination port PIDs/comms found: "
"Time: 6 Type: B Source PID: 7 "
"Source comm: comm2 Source port : 4 "
"Dest (port, comm) pairs: "
"[('pid2', 'test2'), ('pid3', 'test3')] "
"Net namespace: 5")
]
output_mock.error_.assert_has_calls(expected_errors)
def _get_generator(self, raw_data):
""" Convert raw data to standard datatypes and yield it """
for event_data in raw_data:
# e.g. perf a 6997 [003] 363654.881950: sched:sched_wakeup:
event_data = event_data.strip()
match = re.match(
r"\s*"
r"(?P<name>\S+(\s+\S+)*)\s+"
r"(?P<pid>\d+)\s+"
r"\[(?P<cpu>\d+)\]\s+"
r"(?P<time>\d+.\d+):\s+"
r"(?P<event>\S+)", event_data)
# If it did not match, log it but continue
if match is None:
logger.error(
"Failed to parse event data: %s Expected "
"format: name pid cpu time event", event_data)
continue
# Convert time format to us. Perf output: [seconds].[us]
time_str = match.group("time").split(".")
time_int = int(time_str[0]) * 1000000 + int(time_str[1])
specific_datum = {
'pid': match.group("pid"),
'comm': match.group('name'),
'cpu': match.group('cpu')
}
# Connected is none to specify we have a standalone event with no
# connections
event = data_io.EventDatum(specific_datum=specific_datum,
time=time_int,
connected=None,
type=match.group("event"))
yield event
def _generate_events(self, data, port_lookup):
"""
Generate EventDatum objects using tcptracer data and the port-mapping
generated from that data by _generate_dict
:param data:
The raw tcptracer data.
:param port_lookup:
The port-mapping dictionary generated from the raw data by
_generate_dict
:return:
A generator of :class:`EventDatum` objects.
"""
# Skip header
data.seek(0)
data.readline()
data.readline()
for line in data:
values = line.split()
time = int(values[0])
tcp_type = values[1] # connect, accept, close, send or recv
source_pid = int(values[2])
source_comm = values[3]
source_addr = values[5]
dest_addr = values[6]
source_port = int(values[7])
dest_port = int(values[8])
size = int(values[9])
net_ns = int(values[10])
# Discard external TCP
if not source_addr.startswith("127.") or \
not dest_addr.startswith("127.") or \
self.options and self.options.net_ns != net_ns:
continue
# Get destination PIDs from port_lookup dictionary
if dest_port not in port_lookup:
output.error_(
text="IPC: Could not find destination port PID/comm. "
"Check log for details.",
description="Could not find destination port PID/comm: "
"Time: {} Type: {} Source PID: {} "
"Source comm: {} Source port : {} "
"Dest port: {} Net namespace: {}".format(
time, tcp_type, source_pid, source_comm, source_port,
dest_port, net_ns))
continue
dest_pids = port_lookup[dest_port]
# Drop if there are multiple possible PIDs
if len(dest_pids) != 1:
output.error_(
text="IPC: Too many destination port PIDs/comms found. "
"Check log for details.",
description="Too many destination port PIDs/comms found: "
"Time: {} Type: {} Source PID: {} "
"Source comm: {} Source port : {} "
"Dest (port, comm) pairs: {} Net namespace: {}".format(
time, tcp_type, source_pid, source_comm, source_port,
str(sorted(dest_pids)), net_ns))
continue
dest_pid, dest_comm = dest_pids.pop()
dest_pids.add((dest_pid, dest_comm)) # Ensure set isn't altered
# Otherwise output event
event = data_io.EventDatum(time=time,
type=tcp_type,
specific_datum={
"source_pid": source_pid,
"source_comm": source_comm,
"source_port": source_port,
"dest_pid": dest_pid,
"dest_comm": dest_comm,
"dest_port": dest_port,
"size": size,
"net_ns": net_ns
},
connected=[('source_', 'dest_')])
yield event