def test_recv_ipfix_packet(self):
"""
Test general sending of raw and receiving and parsing of these packets.
If this test runs successfully, the sender thread has sent a raw bytes packet towards a locally
listening collector thread, and the collector has successfully received and parsed the packets.
:return:
"""
# send packet without any template, must fail to parse (packets are queued)
pkts, _, _ = send_recv_packets([PACKET_IPFIX])
self.assertEqual(len(pkts),
0) # no export is parsed due to missing template
# send packet with 5 templates and 20 flows, should parse correctly since the templates are known
pkts, _, _ = send_recv_packets([PACKET_IPFIX_TEMPLATE])
self.assertEqual(len(pkts), 1)
p = pkts[0]
self.assertEqual(p.client[0], "127.0.0.1")
self.assertEqual(len(p.export.flows),
1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) # count flows
self.assertEqual(len(p.export.templates), 4 + 1) # count new templates
# send template and multiple export packets
pkts, _, _ = send_recv_packets(
[PACKET_IPFIX, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX])
self.assertEqual(len(pkts), 3)
self.assertEqual(pkts[0].export.header.version, 10)
# check amount of flows across all packets
total_flows = 0
for packet in pkts:
total_flows += len(packet.export.flows)
self.assertEqual(total_flows,
2 + 1 + (1 + 2 + 2 + 9 + 1 + 2 + 1 + 2) + 2 + 1)
def test_compare_memory(self):
"""
Test memory usage of two collector runs with IPFIX and NetFlow v9 packets respectively.
Then compare the two memory snapshots to make sure the libraries do not cross each other.
TODO: more features could be tested, e.g. too big of a difference if one version is optimized better
:return:
"""
pkts, t1, t2 = send_recv_packets(
generate_packets(NUM_PACKETS_PERFORMANCE, 10))
self.assertEqual(len(pkts), NUM_PACKETS_PERFORMANCE)
snapshot_ipfix = tracemalloc.take_snapshot()
del pkts
tracemalloc.clear_traces()
pkts, t1, t2 = send_recv_packets(
generate_packets(NUM_PACKETS_PERFORMANCE, 9))
self.assertEqual(len(pkts), NUM_PACKETS_PERFORMANCE)
snapshot_v9 = tracemalloc.take_snapshot()
del pkts
stats = snapshot_v9.compare_to(snapshot_ipfix, "lineno")
for stat in stats:
if stat.traceback[0].filename.endswith("netflow/ipfix.py"):
self.assertEqual(stat.count, 0)
self.assertEqual(stat.size, 0)
stats = snapshot_ipfix.compare_to(snapshot_v9, "lineno")
for stat in stats:
if stat.traceback[0].filename.endswith("netflow/v9.py"):
self.assertEqual(stat.count, 0)
self.assertEqual(stat.size, 0)
def _test_recv_all_packets(self, num, template_idx, delay=0.0001):
"""Fling packets at the server and test that it receives them all"""
def gen_pkts(n, idx):
for x in range(n):
if x == idx:
yield PACKET_V9_TEMPLATE
else:
yield random.choice(PACKETS_V9)
pkts, tstart, tend = send_recv_packets(gen_pkts(num, template_idx),
delay=delay)
# check number of packets
self.assertEqual(len(pkts), num)
# check timestamps are when packets were sent, not processed
self.assertTrue(all(tstart < p.ts < tend for p in pkts))
# check number of "things" in the packets (flows + templates)
# template packet = 10 things
# other packets = 12 things
self.assertEqual(sum(p.export.header.count for p in pkts),
(num - 1) * 12 + 10)
# check number of flows in the packets
# template packet = 8 flows (2 templates)
# other packets = 12 flows
self.assertEqual(sum(len(p.export.flows) for p in pkts),
(num - 1) * 12 + 8)
def test_recv_v5_packet(self):
"""Test NetFlow v5 packet parsing"""
pkts, _, _ = send_recv_packets([PACKET_V5])
self.assertEqual(len(pkts), 1)
p = pkts[0]
self.assertEqual(p.client[0], "127.0.0.1")
self.assertEqual(len(p.export.flows),
3) # ping request and reply, one multicast
self.assertEqual(p.export.header.count, 3)
self.assertEqual(p.export.header.version, 5)
# Check specific IP address contained in a flow.
# Since it might vary which flow of the pair is epxorted first, check both
flow = p.export.flows[0]
self.assertIn(
ipaddress.ip_address(
flow.IPV4_SRC_ADDR
), # convert to ipaddress obj because value is int
[
ipaddress.ip_address("172.17.0.1"),
ipaddress.ip_address("172.17.0.2")
] # matches multicast packet too
)
self.assertEqual(flow.PROTO, 1) # ICMP
def test_ipfix_contents(self):
"""
Inspect content of exported flows, eg. test the value of an option flow and the correct
parsing of IPv4 and IPv6 addresses.
:return:
"""
p = send_recv_packets([PACKET_IPFIX_TEMPLATE])[0][0]
flow = p.export.flows[0]
self.assertEqual(flow.meteringProcessId, 2649)
self.assertEqual(flow.selectorAlgorithm, 1)
self.assertEqual(flow.systemInitTimeMilliseconds, 1585735165729)
flow = p.export.flows[1] # HTTPS flow from web server to client
self.assertEqual(flow.destinationIPv4Address, 2886795266)
self.assertEqual(ipaddress.ip_address(flow.destinationIPv4Address),
ipaddress.ip_address("172.17.0.2"))
self.assertEqual(flow.protocolIdentifier, 6) # TCP
self.assertEqual(flow.sourceTransportPort, 443)
self.assertEqual(flow.destinationTransportPort, 57766)
self.assertEqual(flow.tcpControlBits, 0x1b)
flow = p.export.flows[17] # IPv6 flow
self.assertEqual(flow.protocolIdentifier, 17) # UDP
self.assertEqual(flow.sourceIPv6Address,
0xfde66f14e0f196090000affeaffeaffe)
self.assertEqual(
ipaddress.ip_address(flow.sourceIPv6Address), # Docker ULA
ipaddress.ip_address("fde6:6f14:e0f1:9609:0:affe:affe:affe"))
def test_ignore_invalid_packets(self):
"""Test that invalid packets log a warning but are otherwise ignored"""
with self.assertLogs(level='WARNING'):
pkts, _, _ = send_recv_packets([
PACKET_INVALID, PACKET_V9_TEMPLATE,
random.choice(PACKETS_V9), PACKET_INVALID,
random.choice(PACKETS_V9), PACKET_INVALID
])
self.assertEqual(len(pkts), 3)
def test_recv_v9_packet(self):
"""Test NetFlow v9 packet parsing"""
# send packet without any template, must fail to parse (packets are queued)
pkts, _, _ = send_recv_packets([PACKETS_V9[0]])
self.assertEqual(len(pkts),
0) # no export is parsed due to missing template
# send packet with two templates and eight flows, should parse correctly since the templates are known
pkts, _, _ = send_recv_packets([PACKET_V9_TEMPLATE])
self.assertEqual(len(pkts), 1)
# and again, but with the templates at the end in the packet
pkts, _, _ = send_recv_packets([PACKET_V9_TEMPLATE_MIXED])
self.assertEqual(len(pkts), 1)
p = pkts[0]
self.assertEqual(p.client[0], "127.0.0.1")
self.assertEqual(len(p.export.flows), 8) # count flows
self.assertEqual(len(p.export.templates), 2) # count new templates
# Inspect contents of specific flows
flow = p.export.flows[0]
self.assertEqual(flow.PROTOCOL, 6) # TCP
self.assertEqual(flow.L4_SRC_PORT, 80)
self.assertEqual(flow.IPV4_SRC_ADDR, "127.0.0.1")
flow = p.export.flows[-1] # last flow
self.assertEqual(flow.PROTOCOL, 17) # UDP
self.assertEqual(flow.L4_DST_PORT, 53)
# send template and multiple export packets
pkts, _, _ = send_recv_packets([PACKET_V9_TEMPLATE, *PACKETS_V9])
self.assertEqual(len(pkts), 4)
self.assertEqual(pkts[0].export.header.version, 9)
# check amount of flows across all packets
total_flows = 0
for packet in pkts:
total_flows += len(packet.export.flows)
self.assertEqual(total_flows, 8 + 12 + 12 + 12)
def _memory_of_version(self,
version,
store_packets=500) -> tracemalloc.Snapshot:
"""
Create memory snapshot of collector run with packets of version :version:
:param version:
:return:
"""
if not tracemalloc.is_tracing():
raise RuntimeError
pkts, t1, t2 = send_recv_packets(generate_packets(
NUM_PACKETS_PERFORMANCE, version),
store_packets=store_packets)
self.assertEqual(len(pkts), NUM_PACKETS_PERFORMANCE)
snapshot = tracemalloc.take_snapshot()
del pkts
return snapshot
def test_time_ipfix(self):
"""
Profile function calls and CPU time.
TODO: this does not work with threading in the collector, yet
:return:
"""
profile = cProfile.Profile()
profile.enable(subcalls=True, builtins=True)
pkts, t1, t2 = send_recv_packets(generate_packets(
NUM_PACKETS_PERFORMANCE, 10),
delay=0,
store_packets=500)
self.assertEqual(len(pkts), NUM_PACKETS_PERFORMANCE)
profile.disable()
for sort_by in ['cumulative', 'calls']:
s = io.StringIO()
ps = pstats.Stats(profile, stream=s)
ps.sort_stats(sort_by).print_stats("netflow")
ps.sort_stats(sort_by).print_callees(.5)
print(s.getvalue())
def test_ipfix_contents_ether(self):
"""
IPFIX content tests based on exports with the softflowd "-T ether" flag, meaning that layer 2
is included in the export, like MAC addresses.
:return:
"""
pkts, _, _ = send_recv_packets(
[PACKET_IPFIX_TEMPLATE_ETHER, PACKET_IPFIX_ETHER])
self.assertEqual(len(pkts), 2)
p = pkts[0]
# Inspect contents of specific flows
flow = p.export.flows[0]
self.assertEqual(flow.meteringProcessId, 9)
self.assertEqual(flow.selectorAlgorithm, 1)
self.assertEqual(flow.systemInitTimeMilliseconds, 759538800000)
flow = p.export.flows[1]
self.assertEqual(flow.destinationIPv4Address, 2886795266)
self.assertTrue(hasattr(flow, "sourceMacAddress"))
self.assertTrue(hasattr(flow, "postDestinationMacAddress"))
self.assertEqual(flow.sourceMacAddress, 0x123456affefe)
self.assertEqual(flow.postDestinationMacAddress, 0xaffeaffeaffe)
def test_recv_v1_packet(self):
"""Test NetFlow v1 packet parsing"""
pkts, _, _ = send_recv_packets([PACKET_V1])
self.assertEqual(len(pkts), 1)
# Take the parsed packet and check meta data
p = pkts[0]
self.assertEqual(p.client[0], "127.0.0.1") # collector listens locally
self.assertEqual(len(p.export.flows), 2) # ping request and reply
self.assertEqual(p.export.header.count, 2) # same value, in header
self.assertEqual(p.export.header.version, 1)
# Check specific IP address contained in a flow.
# Since it might vary which flow of the pair is epxorted first, check both
flow = p.export.flows[0]
self.assertIn(
ipaddress.ip_address(
flow.IPV4_SRC_ADDR
), # convert to ipaddress obj because value is int
[
ipaddress.ip_address("172.17.0.1"),
ipaddress.ip_address("172.17.0.2")
])
self.assertEqual(flow.PROTO, 1) # ICMP