# coding=utf-8 from utils.script_functions import set_parameter from os import system if __name__ == "__main__": parameters_filepath = "parameters.ini" # Keep as all 5 train_methods = ['natural', 'rfgsm_k', 'dfgsm_k', 'bga_k', 'bca_k', 'grosse'] evasion_methods = ['natural', 'rfgsm_k', 'dfgsm_k', 'bga_k', 'bca_k', 'grosse'] for train_method in train_methods: set_parameter(parameters_filepath, "general", "train_model_from_scratch", "True") set_parameter(parameters_filepath, "general", "load_model_weights", "False") set_parameter(parameters_filepath, "general", "experiment_suffix", "run_experiments") set_parameter(parameters_filepath, "general", "training_method", train_method) set_parameter(parameters_filepath, "general", "evasion_method", train_method) system("source activate nn_mal;python framework.py") for train_method in train_methods: model_filepath = "./helper_files/[training:{train_meth}|evasion:{train_meth}]_run_experiments-model.pt".format( train_meth=train_method) set_parameter(parameters_filepath, "general", "training_method", train_method) set_parameter(parameters_filepath, "general", "train_model_from_scratch", "False") set_parameter(parameters_filepath, "general", "load_model_weights", "True") set_parameter(parameters_filepath, "general", "model_weights_path", model_filepath) for evasion_method in evasion_methods:
def eval_defend_submissions(): parameters_filepath = "./helper_files/parameters_holdout.ini" evasion_methods = ['natural', 'rfgsm_k', 'topk', 'bca_k', 'grosse'] for model_filepath in glob.glob(os.path.join(DEFEND_DIR, '*.pt')): submission_name = re.search( "\[training:.*\|", model_filepath).group(0)[:-1].split(':')[-1] print("Evaluating {}'s submission".format(submission_name)) set_parameter(parameters_filepath, "general", "experiment_suffix", submission_name) set_parameter(parameters_filepath, "general", "training_method", submission_name) set_parameter(parameters_filepath, "general", "load_model_weights", "True") set_parameter(parameters_filepath, "general", "model_weights_path", model_filepath) set_parameter(parameters_filepath, "challenge", "eval", "True") set_parameter(parameters_filepath, "challenge", "defend", "True") set_parameter(parameters_filepath, "challenge", "attack", "False") for evasion_method in evasion_methods: set_parameter(parameters_filepath, "general", "evasion_method", evasion_method) main(config_file=parameters_filepath) create_tex_tables(filespath="./result_files")
from utils.script_functions import set_parameter from os import system from framework import main if __name__ == "__main__": parameters_filepath = "parameters.ini" # Keep as all 5 train_methods = ['natural', 'rfgsm_k', 'dfgsm_k', 'bga_k', 'bca_k', 'grosse'] evasion_methods = ['natural', 'rfgsm_k', 'dfgsm_k', 'bga_k', 'bca_k', 'grosse'] experiment_suffix = "demo" # any name you like in case your run multiple experiments # this loop will run adversarial training based on the methods in `train_methods` # it will produce len(train_methods) models storing them at `./helper_files` for train_method in train_methods: set_parameter(parameters_filepath, "general", "load_model_weights", "False") set_parameter(parameters_filepath, "general", "experiment_suffix", experiment_suffix) set_parameter(parameters_filepath, "general", "training_method", train_method) set_parameter(parameters_filepath, "general", "evasion_method", train_method) set_parameter(parameters_filepath, "challenge", "eval", "False") set_parameter(parameters_filepath, "challenge", "defend", "True") set_parameter(parameters_filepath, "challenge", "attack", "False") main() # This loop will fetch the models trained in the above loop # and attack them with the attacks specified in `evasion_methods` for train_method in train_methods: model_filepath = "./helper_files/[training:{train_meth}|evasion:{train_meth}]_{experiment_suffix}-model.pt".format( train_meth=train_method, experiment_suffix=experiment_suffix) set_parameter(parameters_filepath, "general", "experiment_suffix", experiment_suffix)
def eval_attack_submssions(): parameters_filepath = "./helper_files/parameters_attack.ini" model_filepath = "./helper_files/[training:natural|evasion:dfgsm_k]_demo-model.pt" for attack_filepath in glob.glob(os.path.join(ATTACK_DIR, '*.npy')): if attack_filepath.split('/')[-1] == "aes.npy": continue submission_name = attack_filepath.split('/')[-1].split('_')[-1][:-4] print("Evaluating {}'s submission".format(submission_name)) copyfile(attack_filepath, os.path.join(ATTACK_DIR, 'aes.npy')) set_parameter(parameters_filepath, "general", "experiment_suffix", submission_name) set_parameter(parameters_filepath, "general", "training_method", "secret_model") set_parameter(parameters_filepath, "general", "load_model_weights", "True") set_parameter(parameters_filepath, "general", "model_weights_path", model_filepath) set_parameter(parameters_filepath, "challenge", "eval", "True") set_parameter(parameters_filepath, "challenge", "defend", "False") set_parameter(parameters_filepath, "challenge", "attack", "True") set_parameter(parameters_filepath, "general", "evasion_method", submission_name) set_parameter(parameters_filepath, "challenge", "adv_examples_path", ATTACK_DIR) main(config_file=parameters_filepath) create_tex_tables(filespath="./result_files") print("Check evasion latex table file in results_files")