def run(self): offline_streamer = NFStreamer(source=self._path, statistical_analysis=True, splt_analysis=10) print(self._directory) Path(f"{self._directory}").mkdir(parents=True, exist_ok=True) total_flows = offline_streamer.to_csv( flows_per_file=10000, path=f"{self._directory}{self._file_name}.csv")
def test_noroot_live(self): print("\n----------------------------------------------------------------------") try: if sys.platform == 'darwin': streamer_test = NFStreamer(source="lo0", idle_timeout=0) else: streamer_test = NFStreamer(source="lo", idle_timeout=0) except ValueError: print("{}\t: \033[94mOK\033[0m".format(".Testing live capture (noroot)".ljust(60, ' ')))
def test_noroot_live(self): print("\n----------------------------------------------------------------------") print(".Testing live capture (noroot)") try: streamer_test = NFStreamer(idle_timeout=0) except SystemExit: print("PASS.")
def test_unfound_device(self): print("\n----------------------------------------------------------------------") print(".Testing unfoud device") try: streamer_test = NFStreamer(source="inexisting_file.pcap") except SystemExit: print("PASS.")
def save_stream_to_db(self): while True: online_streamer = NFStreamer(source=self.inet, statistical_analysis=True, splt_analysis=10, n_dissections=30) for f in online_streamer: key = f.keys() val = f.values() flow = dict(zip(key, val)) flow = pd.DataFrame([flow]) flow.replace(r'^\s*$', 'unknown', regex=True, inplace=True) for col in flow.columns: if isinstance(flow[col][0], list): colID = 1 for i in flow[col][0]: flow[f'{col}_item_{colID}'] = i colID += 1 flow.drop(columns=[col], inplace=True) flow.drop(columns=['id', 'expiration_id'], inplace=True) flow['timestamp'] = datetime.datetime.now().replace( microsecond=0) flow['resolv_dst'] = flow.dst_ip.apply( lambda ip: self._val_ip(ip, 'reverse_dns')) flow['resolv_src'] = flow.src_ip.apply( lambda ip: self._val_ip(ip, 'reverse_dns')) flow.to_sql(name=self.table_name, con=self.engine, if_exists='append', index=False)
def capture(self): #captura = NFStreamer(source='./Pcap_Files/webex-matheus.pcapng').to_pandas() captura = NFStreamer( source='/home/igor/UMinho/MCT/captura/04-29_hang-spo-you-what.pcapng' ).to_pandas() #captura = NFStreamer(source='/home/igor/UMinho/MCT/mct_tp3/test.pcap').to_pandas() self.data = pd.DataFrame(captura) # converte em dataframe
def test_no_unknown_protocols_without_timeouts(self): files = get_files_list("tests/pcap/") ground_truth_ndpi = build_ground_truth_dict("tests/result/") print("\n----------------------------------------------------------------------") print(".Testing on {} applications:".format(len(files))) ok_files = [] ko_files = [] for test_file in files: streamer_test = NFStreamer(source=test_file, idle_timeout=31556952, active_timeout=31556952) test_case_name = test_file.split('/')[-1] result = {} for flow in streamer_test: if flow.application_name != 'Unknown': try: result[flow.application_name]['bytes'] += flow.bidirectional_raw_bytes result[flow.application_name]['flows'] += 1 result[flow.application_name]['pkts'] += flow.bidirectional_packets except KeyError: result[flow.application_name] = {"bytes": flow.bidirectional_raw_bytes, 'flows': 1, 'pkts': flow.bidirectional_packets} if result == ground_truth_ndpi[test_case_name]: ok_files.append(test_case_name) print("{}\t: \033[94mOK\033[0m".format(test_case_name.ljust(60, ' '))) else: ko_files.append(test_case_name) print(dict(sorted(ground_truth_ndpi[test_case_name].items(), key=lambda x: x[0].lower()))) print("********************************") print(dict(sorted(result.items(), key=lambda x: x[0].lower()) ) ) print("{}\t: \033[31mKO\033[0m".format(test_case_name.ljust(60, ' '))) self.assertEqual(len(files), len(ok_files))
def test_fingerprint_extraction(self): print( "\n----------------------------------------------------------------------" ) fingerprint_streamer = NFStreamer(source='tests/pcap/facebook.pcap', statistical_analysis=True, accounting_mode=1, n_meters=int( os.getenv('MAX_NFMETERS', 0))) for flow in fingerprint_streamer: self.assertEqual(flow.application_name, 'TLS.Facebook') self.assertEqual(flow.application_category_name, 'SocialNetwork') self.assertEqual(flow.application_is_guessed, 0) self.assertTrue(flow.requested_server_name in ['facebook.com', 'www.facebook.com']) self.assertTrue(flow.client_fingerprint in [ 'bfcc1a3891601edb4f137ab7ab25b840', '5c60e71f1b8cd40e4d40ed5b6d666e3f' ]) self.assertTrue(flow.server_fingerprint in [ '2d1eb5817ece335c24904f516ad5da12', '96681175a9547081bf3d417f1a572091' ]) del fingerprint_streamer print("{}\t: \033[94mOK\033[0m".format( ".Test fingerprint extraction".ljust(60, ' ')))
def test_no_unknown_protocols_without_timeouts(self): files = get_files_list("tests/pcap/") ground_truth_ndpi = build_ground_truth_dict("tests/result/") self.maxDif = None print( "----------------------------------------------------------------------" ) print(".Testing on {} applications:".format(len(files))) for test_file in files: streamer_test = NFStreamer(source=test_file, idle_timeout=60000, active_timeout=60000) test_case_name = test_file.split('/')[-1] print(test_case_name) result = {} for flow in streamer_test: if flow.application_name != 'Unknown': try: result[flow.application_name] += flow.total_bytes except KeyError: result[flow.application_name] = flow.total_bytes print(result) self.assertEqual(result, ground_truth_ndpi[test_case_name]) del streamer_test print('PASS.')
def test_expiration_management(self): print("\n----------------------------------------------------------------------") streamer_test = NFStreamer(source='tests/pcap/facebook.pcap', idle_timeout=0) flows = [] for flow in streamer_test: flows.append(flow) self.assertEqual(len(flows), 60) print("{}\t: \033[94mOK\033[0m".format(".Testing Streamer expiration management".ljust(60, ' ')))
def test_to_pandas_anonymized(self): print("\n----------------------------------------------------------------------") df = NFStreamer(source='tests/pcap/ethereum.pcap', idle_timeout=31556952, active_timeout=31556952).to_pandas(ip_anonymization=True) self.assertEqual(df.shape[0], 74) self.assertEqual(df.shape[1], 37) print("{}\t: \033[94mOK\033[0m".format(".Testing to Pandas ip_anonymization=True".ljust(60, ' ')))
def test_to_pandas(self): print("\n----------------------------------------------------------------------") df = NFStreamer(source='tests/pcap/facebook.pcap', statistics=True, bpf_filter="src port 52066 or dst port 52066").to_pandas(ip_anonymization=False) self.assertEqual(df["src_port"][0], 52066) self.assertEqual(df.shape[0], 1) self.assertEqual(df.shape[1], 97) print("{}\t: \033[94mOK\033[0m".format(".Testing to Pandas".ljust(60, ' ')))
def test_bad_observer_args(self): print("\n----------------------------------------------------------------------") try: streamer_test = NFStreamer(source=1, promisc=53, snaplen="wrong", bpf_filter=False, decode_tunnels=22) except ValueError as e: self.assertEqual(1, 1) print("{}\t: \033[94mOK\033[0m".format(".Testing parameters handling".ljust(60, ' ')))
def test_expiration_management(self): print("\n----------------------------------------------------------------------") print(".Testing Streamer expiration management:") streamer_test = NFStreamer(source='tests/facebook.pcap', active_timeout=0) flows = [] for flow in streamer_test: flows.append(flow) self.assertEqual(len(flows), 60) print('PASS.')
def test_noroot_live(self): print( "\n----------------------------------------------------------------------" ) try: streamer_test = NFStreamer(source="lo", idle_timeout=0) except SystemExit: print("{}\t: \033[94mOK\033[0m".format( ".Testing live capture (noroot)".ljust(60, ' ')))
def test_unfound_device(self): print( "\n----------------------------------------------------------------------" ) try: streamer_test = NFStreamer(source="inexisting_file.pcap") except SystemExit: print("{}\t: \033[94mOK\033[0m".format( ".Testing unfoud device".ljust(60, ' ')))
def test_expiration_management(self): print( "\n----------------------------------------------------------------------" ) # Idle expiration streamer_expiration = NFStreamer(source='tests/pcap/google_ssl.pcap', idle_timeout=0, n_meters=int( os.getenv('MAX_NFMETERS', 0))) last_id = 0 for flow in streamer_expiration: last_id = flow.id self.assertEqual(last_id, 27) # Active expiration streamer_expiration = NFStreamer(source='tests/pcap/google_ssl.pcap', active_timeout=0, n_meters=int( os.getenv('MAX_NFMETERS', 0))) last_id = 0 for flow in streamer_expiration: last_id = flow.id self.assertEqual(last_id, 27) # Custom expiration streamer_expiration = NFStreamer(source='tests/pcap/google_ssl.pcap', udps=FlowSlicer(limit=1), n_meters=int( os.getenv('MAX_NFMETERS', 0))) last_id = 0 for flow in streamer_expiration: last_id = flow.id self.assertEqual(last_id, 27) streamer_expiration = NFStreamer(source='tests/pcap/google_ssl.pcap', udps=FlowSlicer(limit=4), n_meters=int( os.getenv('MAX_NFMETERS', 0))) last_id = 0 for flow in streamer_expiration: last_id = flow.id self.assertEqual(last_id, 6) print("{}\t: \033[94mOK\033[0m".format( ".Test expiration management".ljust(60, ' ')))
def capture(self): #captura = NFStreamer(source='./Pcap_Files/webex-matheus.pcapng').to_pandas() captura = NFStreamer( source='/home/igor/UMinho/MCT/captura/04-29_hang-spo-you-what.pcapng' ).to_pandas() #captura = NFStreamer(source='/home/igor/UMinho/MCT/mct_tp3/test.pcap').to_pandas() self.data = pd.DataFrame(captura) # converte em dataframe #pd.set_option('display.max_rows',320) print("\nA dimensão do arquivo é: ") print(self.data.shape)
def test_flow_str_representation(self): print("\n----------------------------------------------------------------------") print(".Testing Flow string representation:") streamer_test = NFStreamer(source='tests/facebook.pcap') flows = [] for flow in streamer_test: flows.append(flow) del streamer_test print(flows[0]) print('PASS.')
def test_raw_feature_parsing(self): streamer = NFStreamer( source='tests/pcap/skype.pcap', idle_timeout=60, active_timeout=60, plugins=[bidirectional_packets_matrix(packet_limit=5)], statistics=False ) for entry in streamer: assert isinstance(entry.bidirectional_packets_matrix, np.ndarray) assert entry.bidirectional_packets_matrix.shape[1] == 6
def test_export(self): print( "\n----------------------------------------------------------------------" ) df = NFStreamer(source='tests/steam.pcap', statistical_analysis=True, n_meters=int(os.getenv('MAX_NFMETERS', 0)), n_dissections=20).to_pandas() df_anon = NFStreamer(source='tests/steam.pcap', statistical_analysis=True, n_meters=int(os.getenv('MAX_NFMETERS', 0)), n_dissections=20).to_pandas( columns_to_anonymize=["src_ip", "dst_ip"]) self.assertEqual(df_anon.shape[0], df.shape[0]) self.assertEqual(df_anon.shape[1], df.shape[1]) self.assertEqual(df_anon['src_ip'].nunique(), df['src_ip'].nunique()) self.assertEqual(df_anon['dst_ip'].nunique(), df['dst_ip'].nunique()) total_flows = NFStreamer(source='tests/steam.pcap', statistical_analysis=True, n_meters=int(os.getenv('MAX_NFMETERS', 0)), n_dissections=20).to_csv() df_from_csv = pd.read_csv('tests/steam.pcap.csv') total_flows_anon = NFStreamer(source='tests/steam.pcap', statistical_analysis=True, n_meters=int(os.getenv( 'MAX_NFMETERS', 0)), n_dissections=20).to_csv() df_anon_from_csv = pd.read_csv('tests/steam.pcap.csv') os.remove('tests/steam.pcap.csv') self.assertEqual(total_flows, total_flows_anon) self.assertEqual(total_flows, df_from_csv.shape[0]) self.assertEqual(total_flows_anon, df_anon_from_csv.shape[0]) self.assertEqual(total_flows, df.shape[0]) self.assertEqual(total_flows_anon, df_anon.shape[0]) print("{}\t: \033[94mOK\033[0m".format(".Test export interfaces".ljust( 60, ' ')))
def test_dhcp(self): print( "\n----------------------------------------------------------------------" ) dhcp_df = NFStreamer(source='tests/pcap/dhcp.pcap', n_dissections=0, n_meters=int(os.getenv('MAX_NFMETERS', 0)), udps=DHCP()).to_pandas().sort_values( by=['src_ip']).reset_index(drop=True) self.assertEqual(dhcp_df["udps.dhcp_55"][0], "1,3,6,42") self.assertEqual(dhcp_df["udps.dhcp_options"][0], "[53, 61, 50, 54, 55]") self.assertEqual(dhcp_df["udps.dhcp_addr"][1], "192.168.0.10") print("{}\t: \033[94mOK\033[0m".format(".Test DHCP plugin".ljust( 60, ' ')))
def test_bpf(self): print( "\n----------------------------------------------------------------------" ) streamer_test = NFStreamer( source='tests/pcap/facebook.pcap', bpf_filter="src port 52066 or dst port 52066", n_meters=int(os.getenv('MAX_NFMETERS', 0))) last_id = 0 for flow in streamer_test: last_id = flow.id self.assertEqual(flow.src_port, 52066) self.assertEqual(last_id, 0) del streamer_test print("{}\t: \033[94mOK\033[0m".format(".Test BPF".ljust(60, ' ')))
def test_bad_observer_args(self): print( "\n----------------------------------------------------------------------" ) try: streamer_test = NFStreamer(source=1, promisc=53, snaplen="wrong", bpf_filter=False, account_ip_padding_size="toto", decode_tunnels=22) except SystemExit as e: self.assertEqual(str(e).count("\n"), 6) print("{}\t: \033[94mOK\033[0m".format( ".Testing parameters handling".ljust(60, ' ')))
def test_source_parameter(self): print( "\n----------------------------------------------------------------------" ) value_errors = 0 source = ["inexisting.pcap", "lo", "lo0", 22] for x in source: try: for flow in NFStreamer(source=x): print(flow) except ValueError: value_errors += 1 self.assertEqual(value_errors, 3) print("{}\t: \033[94mOK\033[0m".format(".Test source parameter".ljust( 60, ' ')))
def test_bpf_filter(self): print("\n----------------------------------------------------------------------") streamer_test = NFStreamer(source='tests/pcap/facebook.pcap', statistics=True, bpf_filter="src port 52066 or dst port 52066") count = 0 for flow in streamer_test: print(flow) print(flow.to_namedtuple()) print(flow.to_json()) count = count + 1 self.assertEqual(flow.src_port, 52066) self.assertEqual(count, 1) del streamer_test print("{}\t: \033[94mOK\033[0m".format(".Testing BPF filtering".ljust(60, ' ')))
def test_udps_parameter(self): print( "\n----------------------------------------------------------------------" ) value_errors = 0 udps = [lambda y: y + 1, "NFPlugin"] for x in udps: try: for flow in NFStreamer(source='tests/google_ssl.pcap', udps=x): print(flow) except ValueError: value_errors += 1 self.assertEqual(value_errors, 2) print("{}\t: \033[94mOK\033[0m".format(".Test udps parameter".ljust( 60, ' ')))
def test_flow_metadata_extraction(self): print("\n----------------------------------------------------------------------") streamer_test = NFStreamer(source='tests/pcap/facebook.pcap', bpf_filter="src port 52066 or dst port 52066") flows = [] for flow in streamer_test: flows.append(flow) del streamer_test self.assertEqual(flows[0].client_info, 'facebook.com') self.assertEqual(flows[0].server_info, '*.facebook.com,*.facebook.net,*.fb.com,*.fbcdn.net,*.fbsbx.com,\ *.m.facebook.com,*.messenger.com,*.xx.fbcdn.net,*.xy.fbcdn.net,*.xz.fbcdn.net,facebook.com,fb.com,\ messenger.com') self.assertEqual(flows[0].client_info, 'facebook.com') self.assertEqual(flows[0].ja3_client, 'bfcc1a3891601edb4f137ab7ab25b840') self.assertEqual(flows[0].ja3_server, '2d1eb5817ece335c24904f516ad5da12') print("{}\t: \033[94mOK\033[0m".format(".Testing metadata extraction".ljust(60, ' ')))
def test_bpf_filter_parameter(self): print( "\n----------------------------------------------------------------------" ) value_errors = 0 bpf_filter = ["my filter", 11] for x in bpf_filter: try: for flow in NFStreamer(source='tests/google_ssl.pcap', bpf_filter=x): print(flow) except ValueError: value_errors += 1 self.assertEqual(value_errors, 1) print("{}\t: \033[94mOK\033[0m".format( ".Test bpf_filter parameter".ljust(60, ' ')))
def test_decode_tunnels_parameter(self): print( "\n----------------------------------------------------------------------" ) value_errors = 0 decode_tunnels = [33, "True"] for x in decode_tunnels: try: for flow in NFStreamer(source='tests/google_ssl.pcap', decode_tunnels=x): print(flow) except ValueError: value_errors += 1 self.assertEqual(value_errors, 2) print("{}\t: \033[94mOK\033[0m".format( ".Test decode_tunnels parameter".ljust(60, ' ')))