def run(self):
offline_streamer = NFStreamer(source=self._path,
statistical_analysis=True,
splt_analysis=10)
print(self._directory)
Path(f"{self._directory}").mkdir(parents=True, exist_ok=True)
total_flows = offline_streamer.to_csv(
flows_per_file=10000,
path=f"{self._directory}{self._file_name}.csv")
def test_noroot_live(self):
print("\n----------------------------------------------------------------------")
try:
if sys.platform == 'darwin':
streamer_test = NFStreamer(source="lo0", idle_timeout=0)
else:
streamer_test = NFStreamer(source="lo", idle_timeout=0)
except ValueError:
print("{}\t: \033[94mOK\033[0m".format(".Testing live capture (noroot)".ljust(60, ' ')))
def test_noroot_live(self):
print("\n----------------------------------------------------------------------")
print(".Testing live capture (noroot)")
try:
streamer_test = NFStreamer(idle_timeout=0)
except SystemExit:
print("PASS.")
def test_unfound_device(self):
print("\n----------------------------------------------------------------------")
print(".Testing unfoud device")
try:
streamer_test = NFStreamer(source="inexisting_file.pcap")
except SystemExit:
print("PASS.")
def save_stream_to_db(self):
while True:
online_streamer = NFStreamer(source=self.inet,
statistical_analysis=True,
splt_analysis=10,
n_dissections=30)
for f in online_streamer:
key = f.keys()
val = f.values()
flow = dict(zip(key, val))
flow = pd.DataFrame([flow])
flow.replace(r'^\s*$', 'unknown', regex=True, inplace=True)
for col in flow.columns:
if isinstance(flow[col][0], list):
colID = 1
for i in flow[col][0]:
flow[f'{col}_item_{colID}'] = i
colID += 1
flow.drop(columns=[col], inplace=True)
flow.drop(columns=['id', 'expiration_id'], inplace=True)
flow['timestamp'] = datetime.datetime.now().replace(
microsecond=0)
flow['resolv_dst'] = flow.dst_ip.apply(
lambda ip: self._val_ip(ip, 'reverse_dns'))
flow['resolv_src'] = flow.src_ip.apply(
lambda ip: self._val_ip(ip, 'reverse_dns'))
flow.to_sql(name=self.table_name,
con=self.engine,
if_exists='append',
index=False)
def capture(self):
#captura = NFStreamer(source='./Pcap_Files/webex-matheus.pcapng').to_pandas()
captura = NFStreamer(
source='/home/igor/UMinho/MCT/captura/04-29_hang-spo-you-what.pcapng'
).to_pandas()
#captura = NFStreamer(source='/home/igor/UMinho/MCT/mct_tp3/test.pcap').to_pandas()
self.data = pd.DataFrame(captura) # converte em dataframe
def test_no_unknown_protocols_without_timeouts(self):
files = get_files_list("tests/pcap/")
ground_truth_ndpi = build_ground_truth_dict("tests/result/")
print("\n----------------------------------------------------------------------")
print(".Testing on {} applications:".format(len(files)))
ok_files = []
ko_files = []
for test_file in files:
streamer_test = NFStreamer(source=test_file, idle_timeout=31556952, active_timeout=31556952)
test_case_name = test_file.split('/')[-1]
result = {}
for flow in streamer_test:
if flow.application_name != 'Unknown':
try:
result[flow.application_name]['bytes'] += flow.bidirectional_raw_bytes
result[flow.application_name]['flows'] += 1
result[flow.application_name]['pkts'] += flow.bidirectional_packets
except KeyError:
result[flow.application_name] = {"bytes": flow.bidirectional_raw_bytes,
'flows': 1, 'pkts': flow.bidirectional_packets}
if result == ground_truth_ndpi[test_case_name]:
ok_files.append(test_case_name)
print("{}\t: \033[94mOK\033[0m".format(test_case_name.ljust(60, ' ')))
else:
ko_files.append(test_case_name)
print(dict(sorted(ground_truth_ndpi[test_case_name].items(), key=lambda x: x[0].lower())))
print("********************************")
print(dict(sorted(result.items(),
key=lambda x: x[0].lower())
)
)
print("{}\t: \033[31mKO\033[0m".format(test_case_name.ljust(60, ' ')))
self.assertEqual(len(files), len(ok_files))
def test_fingerprint_extraction(self):
print(
"\n----------------------------------------------------------------------"
)
fingerprint_streamer = NFStreamer(source='tests/pcap/facebook.pcap',
statistical_analysis=True,
accounting_mode=1,
n_meters=int(
os.getenv('MAX_NFMETERS', 0)))
for flow in fingerprint_streamer:
self.assertEqual(flow.application_name, 'TLS.Facebook')
self.assertEqual(flow.application_category_name, 'SocialNetwork')
self.assertEqual(flow.application_is_guessed, 0)
self.assertTrue(flow.requested_server_name in
['facebook.com', 'www.facebook.com'])
self.assertTrue(flow.client_fingerprint in [
'bfcc1a3891601edb4f137ab7ab25b840',
'5c60e71f1b8cd40e4d40ed5b6d666e3f'
])
self.assertTrue(flow.server_fingerprint in [
'2d1eb5817ece335c24904f516ad5da12',
'96681175a9547081bf3d417f1a572091'
])
del fingerprint_streamer
print("{}\t: \033[94mOK\033[0m".format(
".Test fingerprint extraction".ljust(60, ' ')))
def test_no_unknown_protocols_without_timeouts(self):
files = get_files_list("tests/pcap/")
ground_truth_ndpi = build_ground_truth_dict("tests/result/")
self.maxDif = None
print(
"----------------------------------------------------------------------"
)
print(".Testing on {} applications:".format(len(files)))
for test_file in files:
streamer_test = NFStreamer(source=test_file,
idle_timeout=60000,
active_timeout=60000)
test_case_name = test_file.split('/')[-1]
print(test_case_name)
result = {}
for flow in streamer_test:
if flow.application_name != 'Unknown':
try:
result[flow.application_name] += flow.total_bytes
except KeyError:
result[flow.application_name] = flow.total_bytes
print(result)
self.assertEqual(result, ground_truth_ndpi[test_case_name])
del streamer_test
print('PASS.')
def test_expiration_management(self):
print("\n----------------------------------------------------------------------")
streamer_test = NFStreamer(source='tests/pcap/facebook.pcap', idle_timeout=0)
flows = []
for flow in streamer_test:
flows.append(flow)
self.assertEqual(len(flows), 60)
print("{}\t: \033[94mOK\033[0m".format(".Testing Streamer expiration management".ljust(60, ' ')))
def test_to_pandas_anonymized(self):
print("\n----------------------------------------------------------------------")
df = NFStreamer(source='tests/pcap/ethereum.pcap',
idle_timeout=31556952,
active_timeout=31556952).to_pandas(ip_anonymization=True)
self.assertEqual(df.shape[0], 74)
self.assertEqual(df.shape[1], 37)
print("{}\t: \033[94mOK\033[0m".format(".Testing to Pandas ip_anonymization=True".ljust(60, ' ')))
def test_to_pandas(self):
print("\n----------------------------------------------------------------------")
df = NFStreamer(source='tests/pcap/facebook.pcap', statistics=True,
bpf_filter="src port 52066 or dst port 52066").to_pandas(ip_anonymization=False)
self.assertEqual(df["src_port"][0], 52066)
self.assertEqual(df.shape[0], 1)
self.assertEqual(df.shape[1], 97)
print("{}\t: \033[94mOK\033[0m".format(".Testing to Pandas".ljust(60, ' ')))
def test_bad_observer_args(self):
print("\n----------------------------------------------------------------------")
try:
streamer_test = NFStreamer(source=1, promisc=53, snaplen="wrong", bpf_filter=False, decode_tunnels=22)
except ValueError as e:
self.assertEqual(1, 1)
print("{}\t: \033[94mOK\033[0m".format(".Testing parameters handling".ljust(60, ' ')))
def test_expiration_management(self):
print("\n----------------------------------------------------------------------")
print(".Testing Streamer expiration management:")
streamer_test = NFStreamer(source='tests/facebook.pcap', active_timeout=0)
flows = []
for flow in streamer_test:
flows.append(flow)
self.assertEqual(len(flows), 60)
print('PASS.')
def test_noroot_live(self):
print(
"\n----------------------------------------------------------------------"
)
try:
streamer_test = NFStreamer(source="lo", idle_timeout=0)
except SystemExit:
print("{}\t: \033[94mOK\033[0m".format(
".Testing live capture (noroot)".ljust(60, ' ')))
def test_unfound_device(self):
print(
"\n----------------------------------------------------------------------"
)
try:
streamer_test = NFStreamer(source="inexisting_file.pcap")
except SystemExit:
print("{}\t: \033[94mOK\033[0m".format(
".Testing unfoud device".ljust(60, ' ')))
def test_expiration_management(self):
print(
"\n----------------------------------------------------------------------"
)
# Idle expiration
streamer_expiration = NFStreamer(source='tests/pcap/google_ssl.pcap',
idle_timeout=0,
n_meters=int(
os.getenv('MAX_NFMETERS', 0)))
last_id = 0
for flow in streamer_expiration:
last_id = flow.id
self.assertEqual(last_id, 27)
# Active expiration
streamer_expiration = NFStreamer(source='tests/pcap/google_ssl.pcap',
active_timeout=0,
n_meters=int(
os.getenv('MAX_NFMETERS', 0)))
last_id = 0
for flow in streamer_expiration:
last_id = flow.id
self.assertEqual(last_id, 27)
# Custom expiration
streamer_expiration = NFStreamer(source='tests/pcap/google_ssl.pcap',
udps=FlowSlicer(limit=1),
n_meters=int(
os.getenv('MAX_NFMETERS', 0)))
last_id = 0
for flow in streamer_expiration:
last_id = flow.id
self.assertEqual(last_id, 27)
streamer_expiration = NFStreamer(source='tests/pcap/google_ssl.pcap',
udps=FlowSlicer(limit=4),
n_meters=int(
os.getenv('MAX_NFMETERS', 0)))
last_id = 0
for flow in streamer_expiration:
last_id = flow.id
self.assertEqual(last_id, 6)
print("{}\t: \033[94mOK\033[0m".format(
".Test expiration management".ljust(60, ' ')))
def capture(self):
#captura = NFStreamer(source='./Pcap_Files/webex-matheus.pcapng').to_pandas()
captura = NFStreamer(
source='/home/igor/UMinho/MCT/captura/04-29_hang-spo-you-what.pcapng'
).to_pandas()
#captura = NFStreamer(source='/home/igor/UMinho/MCT/mct_tp3/test.pcap').to_pandas()
self.data = pd.DataFrame(captura) # converte em dataframe
#pd.set_option('display.max_rows',320)
print("\nA dimensão do arquivo é: ")
print(self.data.shape)
def test_flow_str_representation(self):
print("\n----------------------------------------------------------------------")
print(".Testing Flow string representation:")
streamer_test = NFStreamer(source='tests/facebook.pcap')
flows = []
for flow in streamer_test:
flows.append(flow)
del streamer_test
print(flows[0])
print('PASS.')
def test_raw_feature_parsing(self):
streamer = NFStreamer(
source='tests/pcap/skype.pcap',
idle_timeout=60,
active_timeout=60,
plugins=[bidirectional_packets_matrix(packet_limit=5)],
statistics=False
)
for entry in streamer:
assert isinstance(entry.bidirectional_packets_matrix, np.ndarray)
assert entry.bidirectional_packets_matrix.shape[1] == 6
def test_export(self):
print(
"\n----------------------------------------------------------------------"
)
df = NFStreamer(source='tests/steam.pcap',
statistical_analysis=True,
n_meters=int(os.getenv('MAX_NFMETERS', 0)),
n_dissections=20).to_pandas()
df_anon = NFStreamer(source='tests/steam.pcap',
statistical_analysis=True,
n_meters=int(os.getenv('MAX_NFMETERS', 0)),
n_dissections=20).to_pandas(
columns_to_anonymize=["src_ip", "dst_ip"])
self.assertEqual(df_anon.shape[0], df.shape[0])
self.assertEqual(df_anon.shape[1], df.shape[1])
self.assertEqual(df_anon['src_ip'].nunique(), df['src_ip'].nunique())
self.assertEqual(df_anon['dst_ip'].nunique(), df['dst_ip'].nunique())
total_flows = NFStreamer(source='tests/steam.pcap',
statistical_analysis=True,
n_meters=int(os.getenv('MAX_NFMETERS', 0)),
n_dissections=20).to_csv()
df_from_csv = pd.read_csv('tests/steam.pcap.csv')
total_flows_anon = NFStreamer(source='tests/steam.pcap',
statistical_analysis=True,
n_meters=int(os.getenv(
'MAX_NFMETERS', 0)),
n_dissections=20).to_csv()
df_anon_from_csv = pd.read_csv('tests/steam.pcap.csv')
os.remove('tests/steam.pcap.csv')
self.assertEqual(total_flows, total_flows_anon)
self.assertEqual(total_flows, df_from_csv.shape[0])
self.assertEqual(total_flows_anon, df_anon_from_csv.shape[0])
self.assertEqual(total_flows, df.shape[0])
self.assertEqual(total_flows_anon, df_anon.shape[0])
print("{}\t: \033[94mOK\033[0m".format(".Test export interfaces".ljust(
60, ' ')))
def test_dhcp(self):
print(
"\n----------------------------------------------------------------------"
)
dhcp_df = NFStreamer(source='tests/pcap/dhcp.pcap',
n_dissections=0,
n_meters=int(os.getenv('MAX_NFMETERS', 0)),
udps=DHCP()).to_pandas().sort_values(
by=['src_ip']).reset_index(drop=True)
self.assertEqual(dhcp_df["udps.dhcp_55"][0], "1,3,6,42")
self.assertEqual(dhcp_df["udps.dhcp_options"][0],
"[53, 61, 50, 54, 55]")
self.assertEqual(dhcp_df["udps.dhcp_addr"][1], "192.168.0.10")
print("{}\t: \033[94mOK\033[0m".format(".Test DHCP plugin".ljust(
60, ' ')))
def test_bpf(self):
print(
"\n----------------------------------------------------------------------"
)
streamer_test = NFStreamer(
source='tests/pcap/facebook.pcap',
bpf_filter="src port 52066 or dst port 52066",
n_meters=int(os.getenv('MAX_NFMETERS', 0)))
last_id = 0
for flow in streamer_test:
last_id = flow.id
self.assertEqual(flow.src_port, 52066)
self.assertEqual(last_id, 0)
del streamer_test
print("{}\t: \033[94mOK\033[0m".format(".Test BPF".ljust(60, ' ')))
def test_bad_observer_args(self):
print(
"\n----------------------------------------------------------------------"
)
try:
streamer_test = NFStreamer(source=1,
promisc=53,
snaplen="wrong",
bpf_filter=False,
account_ip_padding_size="toto",
decode_tunnels=22)
except SystemExit as e:
self.assertEqual(str(e).count("\n"), 6)
print("{}\t: \033[94mOK\033[0m".format(
".Testing parameters handling".ljust(60, ' ')))
def test_source_parameter(self):
print(
"\n----------------------------------------------------------------------"
)
value_errors = 0
source = ["inexisting.pcap", "lo", "lo0", 22]
for x in source:
try:
for flow in NFStreamer(source=x):
print(flow)
except ValueError:
value_errors += 1
self.assertEqual(value_errors, 3)
print("{}\t: \033[94mOK\033[0m".format(".Test source parameter".ljust(
60, ' ')))
def test_bpf_filter(self):
print("\n----------------------------------------------------------------------")
streamer_test = NFStreamer(source='tests/pcap/facebook.pcap',
statistics=True,
bpf_filter="src port 52066 or dst port 52066")
count = 0
for flow in streamer_test:
print(flow)
print(flow.to_namedtuple())
print(flow.to_json())
count = count + 1
self.assertEqual(flow.src_port, 52066)
self.assertEqual(count, 1)
del streamer_test
print("{}\t: \033[94mOK\033[0m".format(".Testing BPF filtering".ljust(60, ' ')))
def test_udps_parameter(self):
print(
"\n----------------------------------------------------------------------"
)
value_errors = 0
udps = [lambda y: y + 1, "NFPlugin"]
for x in udps:
try:
for flow in NFStreamer(source='tests/google_ssl.pcap', udps=x):
print(flow)
except ValueError:
value_errors += 1
self.assertEqual(value_errors, 2)
print("{}\t: \033[94mOK\033[0m".format(".Test udps parameter".ljust(
60, ' ')))
def test_flow_metadata_extraction(self):
print("\n----------------------------------------------------------------------")
streamer_test = NFStreamer(source='tests/pcap/facebook.pcap', bpf_filter="src port 52066 or dst port 52066")
flows = []
for flow in streamer_test:
flows.append(flow)
del streamer_test
self.assertEqual(flows[0].client_info, 'facebook.com')
self.assertEqual(flows[0].server_info, '*.facebook.com,*.facebook.net,*.fb.com,*.fbcdn.net,*.fbsbx.com,\
*.m.facebook.com,*.messenger.com,*.xx.fbcdn.net,*.xy.fbcdn.net,*.xz.fbcdn.net,facebook.com,fb.com,\
messenger.com')
self.assertEqual(flows[0].client_info, 'facebook.com')
self.assertEqual(flows[0].ja3_client, 'bfcc1a3891601edb4f137ab7ab25b840')
self.assertEqual(flows[0].ja3_server, '2d1eb5817ece335c24904f516ad5da12')
print("{}\t: \033[94mOK\033[0m".format(".Testing metadata extraction".ljust(60, ' ')))
def test_bpf_filter_parameter(self):
print(
"\n----------------------------------------------------------------------"
)
value_errors = 0
bpf_filter = ["my filter", 11]
for x in bpf_filter:
try:
for flow in NFStreamer(source='tests/google_ssl.pcap',
bpf_filter=x):
print(flow)
except ValueError:
value_errors += 1
self.assertEqual(value_errors, 1)
print("{}\t: \033[94mOK\033[0m".format(
".Test bpf_filter parameter".ljust(60, ' ')))
def test_decode_tunnels_parameter(self):
print(
"\n----------------------------------------------------------------------"
)
value_errors = 0
decode_tunnels = [33, "True"]
for x in decode_tunnels:
try:
for flow in NFStreamer(source='tests/google_ssl.pcap',
decode_tunnels=x):
print(flow)
except ValueError:
value_errors += 1
self.assertEqual(value_errors, 2)
print("{}\t: \033[94mOK\033[0m".format(
".Test decode_tunnels parameter".ljust(60, ' ')))
