def setUpClass(cls): """Initiates the parameters to feed the test functions and previous functions to generated the necessary files.""" gatherer.convert_pcap_nfcapd(pcap_path, pcap_file, nfcapd_path, 60) nfcapd_files = util.directory_content(nfcapd_path)[1] gatherer.convert_nfcapd_csv(nfcapd_path, nfcapd_files, csv_path, 'test') csv_file = util.directory_content(csv_path)[1][0] cls.header, cls.flows = gatherer.open_csv(csv_path, csv_file, 30)
def test_file_creation(self): """Tests if the process started by the function is creating the files correctly.""" # delay one second to allow the start of the process time.sleep(1) self.assertIn('nfcapd.current', util.directory_content(nfcapd_path)[1][0])
def gathering(self, nfcapd_files): gatherer.convert_nfcapd_csv(util.paths['nfcapd'], nfcapd_files, f'{util.paths["csv"]}tmp/', 'realtime') csv_file = util.directory_content(f'{util.paths["csv"]}tmp/')[1] logger.info(f'csv files: {csv_file[0]}') _, flows = gatherer.open_csv(f'{util.paths["csv"]}tmp/', csv_file[0]) return flows
def execution(self): process = gatherer.capture_nfcapd(util.paths['nfcapd'], 60) dataset = Dataset.query.get(self.model.dataset_id) logger.info(f'process pid: {process.pid}') logger.info(f'dataset file: {dataset.file}') try: while not self.event.is_set(): nfcapd_files = util.directory_content(util.paths['nfcapd'])[1] try: if not 'current' in nfcapd_files[0]: logger.info(f'nfcapd files: {nfcapd_files[:-1]}') # gathering flows. flows = self.gathering(nfcapd_files[:-1]) # cleaning remaining files util.clean_directory(util.paths['nfcapd'], 'nfcapd.20*') util.clean_directory(f'{util.paths["csv"]}tmp/', '*') if len(flows[0]) < 18: raise ValueError('No matched flows') logger.info(f'flow: {flows[0]}') # preprocessing flows. formatter = Formatter() flows = formatter.format_flows(flows) logger.info(f'formatted flow: {flows[0]}') modifier = Modifier(2, dataset.aggregation) extractor = Extractor([ feature.id + 7 for feature in self.model.features ]) while flows: flow, flows = modifier.aggregate(flows) features, _ = extractor.extract(flow) # detecting intrusions. pred, _, _ = self.detector.test([features]) if pred[0]: # mitigating intrusions. self.mitigating(flow) time.sleep(2) except IndexError: time.sleep(2) continue except ValueError as error: logger.error(error) util.clean_directory(util.paths['nfcapd'], 'nfcapd.20*') util.clean_directory(f'{util.paths["csv"]}tmp/', '*') continue finally: logger.info('thread status: false') process.kill()
def setUp(self): """Initiates the parameters to feed the test functions and previous functions to generated the necessary files.""" gatherer.convert_pcap_nfcapd(pcap_path, pcap_file, nfcapd_path, 60) nfcapd_files = util.directory_content(nfcapd_path)[1] gatherer.convert_nfcapd_csv(nfcapd_path, nfcapd_files, csv_path, 'test')
def test_last_file_size(self): """Tests if the last file generated has the expected remaining size.""" pcap_files = util.directory_content(f'{pcap_path}split_normal0/')[1] last_file_size = os.path.getsize(f'{pcap_path}split_normal0/' f'{pcap_files[-1]}') # converts the last file size to megabytes and then rounded up self.assertEqual(ceil(last_file_size / (1000**2)), 173, 'different size of the last file')
def setUpClass(cls): """Initiates the parameters to feed the test functions.""" # gathering flows raw_csv_file = util.directory_content(formatter_path)[1][0] header, flows = gatherer.open_csv(formatter_path, raw_csv_file) # preprocessing flows formatter = Formatter() cls.header = formatter.format_header(header) cls.flows = formatter.format_flows(flows)
def test_time_interval(self): """Tests if the time interval between the nfcapd files is according to the defined time.""" nfcapd_files = util.directory_content(nfcapd_path)[1] # gets the minutes in the nfcapd file name minutes = [int(file.split('.')[-1][-2:]) for file in nfcapd_files] # checks if all files have the same time interval for x, y in zip(minutes[:-1], minutes[1:]): self.assertEqual(y - x, 1, 'wrong time interval')
def test_extract_specific_features(self): """Tests if specifics features and labels were correctly extracted from the flows.""" # gathering features expt_csv = util.directory_content(extractor_path)[1][-1] expt_features = gatherer.open_csv(extractor_path, expt_csv)[1] extractor = Extractor([feature + 7 for feature in [1, 3]]) features, labels = extractor.extract_features_labels(self.flows) self.assertListEqual(features, expt_features, 'features extracted incorrectly')
def test_extract_features_labels(self): """Tests if the features and labels were correctly extracted from the flows.""" # gathering features expt_csv = util.directory_content(extractor_path)[1][0] expt_features = gatherer.open_csv(extractor_path, expt_csv)[1] extractor = Extractor([feature + 7 for feature in range(1, 10)]) features, labels = extractor.extract_features_labels(self.flows) self.assertListEqual(features, expt_features, 'features extracted incorrectly') self.assertEqual(labels[0], '0', 'labels extracted incorrectly')
def test_aggregate_flows(self): """Tests if the features were correctly aggregated.""" # gathering flows expt_csv = util.directory_content(modifier_path)[1][0] expt_header, expt_flows = gatherer.open_csv(modifier_path, expt_csv) # preprocessing flows formatter = Formatter(gather=False, train=True) expt_flows = formatter.format_flows(expt_flows) self.assertListEqual(self.header, expt_header, 'aggregation performed incorrectly in header') self.assertListEqual(self.flows, expt_flows, 'aggregation performed incorrectly in flows')
def setUpClass(cls): """Initiates the parameters to feed the test functions.""" # gathering flows raw_csv_file = util.directory_content(modifier_path)[1][-1] header, flows = gatherer.open_csv(modifier_path, raw_csv_file) # preprocessing flows formatter = Formatter() header = formatter.format_header(header) flows = formatter.format_flows(flows) # threshold defined according to the expected result in test dataset modifier = Modifier(label=0, threshold=5) cls.header = modifier.extend_header(header) cls.flows = modifier.aggregate_flows(flows)
def test_file_interval(self): """Tests if the csv file interval generate by the nfcapd files is correct.""" csv_file = util.directory_content(csv_path)[1][0] flows = list() with open(f'{csv_path}{csv_file}') as file: reader = csv.reader(file) for line in reader: flows.append(line) # gets the first and last minute of the flows to check the interval start_time = datetime.strptime(flows[1][0], '%Y-%m-%d %H:%M:%S').minute final_time = datetime.strptime(flows[-4][0], '%Y-%m-%d %H:%M:%S').minute self.assertEqual(final_time - start_time, 13, 'wrong file interval')
def content(function, directory): form = ContentForm(request.form) # clearing the previous directory. if directory in ['pcap', 'nfcapd', 'csv']: paths_hist = {'root': util.root} else: paths_hist = session['paths_hist'] # checking if the folder was already opened. if not directory in paths_hist.keys(): paths_hist[directory] = f'{directory}/' # creating the full path. full_path = paths_hist['root'] + paths_hist[directory] # getting inner directory content. inner_dirs, files = util.directory_content(full_path) form.files_choices(files) if request.method == 'POST' and form.validate_on_submit(): logger.info(f'file: {form.files.data}') session['files'] = form.files.data return redirect(url_for(f'creation.{function}', directory=directory)) # creating the paths of the inner directories. for inner_dir in inner_dirs: paths_hist[inner_dir] = f'{paths_hist[directory]}{inner_dir}/' session['paths_hist'] = paths_hist return render_template('creation/content.html', form=form, inner_dirs=inner_dirs, relative_path=session['paths_hist'][directory], function=function)
def load(): files = util.directory_content(f'{util.paths["models"]}')[1] models = [Model.query.get(file.split('_')[0]) for file in files] return render_template('setting/load.html', models=models)
def datasets_choices(self): datasets = util.directory_content(f'{util.paths["csv"]}datasets/')[1] for ds in datasets: self.dataset.choices.append([ds, ' '.join(ds.split('.csv')[0] .split('_'))])
def test_num_files(self): """Tests the number of files created by splitting the pcap files.""" self.assertEqual( len(util.directory_content(f'{pcap_path}split_normal0/')[1]), 4, 'different number of pcap files')
def test_num_files(self): """Tests if the number of files match if the number of minutes in the pcap file.""" self.assertEqual(len(util.directory_content(nfcapd_path)[1]), 14, 'different number of nfcapd files')
from datetime import datetime from math import ceil base_path = os.path.abspath(os.path.dirname('intrusion_prevention_system')) sys.path.append(base_path) from app.core import gatherer from app.core import util # defines the main paths use during the tests pcap_path = f'{base_path}/tests/app/core/data/gatherer/pcap/' nfcapd_path = f'{base_path}/tests/app/core/data/gatherer/nfcapd/' csv_path = f'{base_path}/tests/app/core/data/gatherer/csv/' # file used in the tests pcap_file = util.directory_content(pcap_path)[1] # unit tests class TestSplitPcap(unittest.TestCase): """Tests the split_pcap function in gatherer module.""" @classmethod def setUpClass(cls): """Initiates the parameters to feed the test functions.""" cls.split_size = 300 # function to be tested gatherer.split_pcap(pcap_path, pcap_file, cls.split_size) @classmethod
def setUpClass(cls): """Initiates the parameters to feed the test functions.""" # gathering flows modified_csv_file = util.directory_content(extractor_path)[1][1] _, cls.flows = gatherer.open_csv(extractor_path, modified_csv_file)