def run_models(algorithm, similarity_score, test_data_path, results_path,
new_model_running, event):
"""
:param algorithm:
:param similarity_score:
:param test_data_path:
:param results_path:
:param new_model_running:
:param event:
:return:
"""
if new_model_running:
training_data_path, save_model, algorithms, threshold, features_list, target_features_list = ModelsExecution.get_new_model_parameters(
)
else:
training_data_path, save_model, algorithms, threshold = ModelsExecution.get_load_model_parameters(
)
# Set new nested dictionary for a chosen algorithm
results_data = InputSettings.get_results_metrics_data()
results_data[algorithm] = dict()
InputSettings.update_results_metrics_data(results_data)
# Checks whether the current flow in the system is new model creation or loading an existing model
if new_model_running:
algorithm_model_path = None
algorithm_features_list = features_list[algorithm]
algorithm_target_features_list = target_features_list[algorithm]
train_scaler_path = None
target_scaler_path = None
else:
algorithm_path = InputSettings.get_existing_algorithm_path(
algorithm)
model_data_json_path = os.path.join(
*[str(algorithm_path), 'model_data.json'])
algorithm_json_file = read_json_file(f"{model_data_json_path}")
algorithm_features_list = algorithm_json_file['features']
algorithm_target_features_list = algorithm_json_file[
'target_features']
threshold = algorithm_json_file['threshold']
algorithm_model_path = get_model_path(algorithm_path)
train_scaler_path = get_scaler_path(algorithm_path, 'train')
target_scaler_path = get_scaler_path(algorithm_path, 'target')
# Dynamic execution for each chosen model
model_execution_function = ModelsExecution.get_algorithm_execution_function(
algorithm)
model_execution_function(
test_data_path, results_path, similarity_score, training_data_path,
save_model, new_model_running, algorithm_model_path, threshold,
algorithm_features_list, algorithm_target_features_list,
train_scaler_path, target_scaler_path, event)
def report_results(results_dir_path,
test_data_path,
FLIGHT_ROUTES,
algorithm_name,
similarity_function,
routes_duration,
attack_duration,
verbose=1):
"""
report all the results, according to the algorithm in the input
:param results_dir_path: the path of results directory
:param test_data_path: the path of test dataset directory
:param FLIGHT_ROUTES: names of existing flight routes
:param algorithm_name: the name of the algorithm that we want to report about
:param similarity_function: the similarity function we currently report about
:param routes_duration: routes time duration
:param attack_duration: attacks time duration
:param verbose: default = 1 , otherwise = can be changed to 0
:return: all the reports are saved to suitable csv files
"""
# Set new nested dictionary for a flight route from all the existing flights routes
from utils.input_settings import InputSettings
results_data = InputSettings.get_results_metrics_data()
# Iterate over all existing flight routes in order to present them in the final results table
for flight_route in FLIGHT_ROUTES:
flight_dir = os.path.join(test_data_path, flight_route)
ATTACKS = get_subdirectories(flight_dir)
try:
results_data[algorithm_name][flight_route]
except KeyError:
results_data[algorithm_name][flight_route] = dict()
results_data[algorithm_name][flight_route][similarity_function] = dict(
)
metrics_list = ['fpr', 'tpr', 'acc', 'delay']
for metric in metrics_list:
results = pd.DataFrame(columns=ATTACKS)
# Iterate over all the flight routes in order to save each results' permutation in a csv file
for i, flight_route in enumerate(FLIGHT_ROUTES):
flight_route_metric_path = os.path.join(*[
str(results_dir_path),
str(flight_route),
str(flight_route) + '_' + str(metric) + '.csv'
])
df = pd.read_csv(f"{flight_route_metric_path}")
mean = df.mean(axis=0).values
std = df.std(axis=0).values
output = []
for x, y in zip(mean, std):
if math.isnan(y):
output.append(f"{round(x, 2)}")
else:
output.append(f"{round(x, 2)}±{round(y, 2)}%")
results.loc[i] = output
results_data[algorithm_name][flight_route][similarity_function][
metric] = dict()
# Iterate over all existing attacks in test data set
for j, attack in enumerate(ATTACKS):
results_data[algorithm_name][flight_route][
similarity_function][metric][attack] = dict()
results_data[algorithm_name][flight_route][
similarity_function][metric][attack] = output[j]
results_data[algorithm_name][flight_route][
attack + '_duration'] = dict()
results_data[algorithm_name][flight_route][attack + '_duration'] = \
routes_duration[attack][0]
results_data[algorithm_name][flight_route][
attack + '_attack_duration'] = dict()
results_data[algorithm_name][flight_route][attack + '_attack_duration'] = \
attack_duration[attack][0]
results.index = FLIGHT_ROUTES
if verbose:
print(results)
# Update evaluated evaluation metric for each attack according to the current algorithm and metric
InputSettings.update_results_metrics_data(results_data)
final_metric_path = os.path.join(
*[str(results_dir_path), 'final_' + str(metric) + '.csv'])
results.to_csv(f"{final_metric_path}")
def get_results_metrics_data(self):
return InputSettings.get_results_metrics_data()