# coding=utf-8
from utils.script_functions import set_parameter
from os import system
if __name__ == "__main__":
parameters_filepath = "parameters.ini"
# Keep as all 5
train_methods = ['natural', 'rfgsm_k', 'dfgsm_k', 'bga_k', 'bca_k', 'grosse']
evasion_methods = ['natural', 'rfgsm_k', 'dfgsm_k', 'bga_k', 'bca_k', 'grosse']
for train_method in train_methods:
set_parameter(parameters_filepath, "general", "train_model_from_scratch", "True")
set_parameter(parameters_filepath, "general", "load_model_weights", "False")
set_parameter(parameters_filepath, "general", "experiment_suffix", "run_experiments")
set_parameter(parameters_filepath, "general", "training_method", train_method)
set_parameter(parameters_filepath, "general", "evasion_method", train_method)
system("source activate nn_mal;python framework.py")
for train_method in train_methods:
model_filepath = "./helper_files/[training:{train_meth}|evasion:{train_meth}]_run_experiments-model.pt".format(
train_meth=train_method)
set_parameter(parameters_filepath, "general", "training_method", train_method)
set_parameter(parameters_filepath, "general", "train_model_from_scratch", "False")
set_parameter(parameters_filepath, "general", "load_model_weights", "True")
set_parameter(parameters_filepath, "general", "model_weights_path", model_filepath)
for evasion_method in evasion_methods:
def eval_defend_submissions():
parameters_filepath = "./helper_files/parameters_holdout.ini"
evasion_methods = ['natural', 'rfgsm_k', 'topk', 'bca_k', 'grosse']
for model_filepath in glob.glob(os.path.join(DEFEND_DIR, '*.pt')):
submission_name = re.search(
"\[training:.*\|", model_filepath).group(0)[:-1].split(':')[-1]
print("Evaluating {}'s submission".format(submission_name))
set_parameter(parameters_filepath, "general", "experiment_suffix",
submission_name)
set_parameter(parameters_filepath, "general", "training_method",
submission_name)
set_parameter(parameters_filepath, "general", "load_model_weights",
"True")
set_parameter(parameters_filepath, "general", "model_weights_path",
model_filepath)
set_parameter(parameters_filepath, "challenge", "eval", "True")
set_parameter(parameters_filepath, "challenge", "defend", "True")
set_parameter(parameters_filepath, "challenge", "attack", "False")
for evasion_method in evasion_methods:
set_parameter(parameters_filepath, "general", "evasion_method",
evasion_method)
main(config_file=parameters_filepath)
create_tex_tables(filespath="./result_files")
from utils.script_functions import set_parameter
from os import system
from framework import main
if __name__ == "__main__":
parameters_filepath = "parameters.ini"
# Keep as all 5
train_methods = ['natural', 'rfgsm_k', 'dfgsm_k', 'bga_k', 'bca_k', 'grosse']
evasion_methods = ['natural', 'rfgsm_k', 'dfgsm_k', 'bga_k', 'bca_k', 'grosse']
experiment_suffix = "demo" # any name you like in case your run multiple experiments
# this loop will run adversarial training based on the methods in `train_methods`
# it will produce len(train_methods) models storing them at `./helper_files`
for train_method in train_methods:
set_parameter(parameters_filepath, "general", "load_model_weights", "False")
set_parameter(parameters_filepath, "general", "experiment_suffix", experiment_suffix)
set_parameter(parameters_filepath, "general", "training_method", train_method)
set_parameter(parameters_filepath, "general", "evasion_method", train_method)
set_parameter(parameters_filepath, "challenge", "eval", "False")
set_parameter(parameters_filepath, "challenge", "defend", "True")
set_parameter(parameters_filepath, "challenge", "attack", "False")
main()
# This loop will fetch the models trained in the above loop
# and attack them with the attacks specified in `evasion_methods`
for train_method in train_methods:
model_filepath = "./helper_files/[training:{train_meth}|evasion:{train_meth}]_{experiment_suffix}-model.pt".format(
train_meth=train_method, experiment_suffix=experiment_suffix)
set_parameter(parameters_filepath, "general", "experiment_suffix", experiment_suffix)
def eval_attack_submssions():
parameters_filepath = "./helper_files/parameters_attack.ini"
model_filepath = "./helper_files/[training:natural|evasion:dfgsm_k]_demo-model.pt"
for attack_filepath in glob.glob(os.path.join(ATTACK_DIR, '*.npy')):
if attack_filepath.split('/')[-1] == "aes.npy": continue
submission_name = attack_filepath.split('/')[-1].split('_')[-1][:-4]
print("Evaluating {}'s submission".format(submission_name))
copyfile(attack_filepath, os.path.join(ATTACK_DIR, 'aes.npy'))
set_parameter(parameters_filepath, "general", "experiment_suffix",
submission_name)
set_parameter(parameters_filepath, "general", "training_method",
"secret_model")
set_parameter(parameters_filepath, "general", "load_model_weights",
"True")
set_parameter(parameters_filepath, "general", "model_weights_path",
model_filepath)
set_parameter(parameters_filepath, "challenge", "eval", "True")
set_parameter(parameters_filepath, "challenge", "defend", "False")
set_parameter(parameters_filepath, "challenge", "attack", "True")
set_parameter(parameters_filepath, "general", "evasion_method",
submission_name)
set_parameter(parameters_filepath, "challenge", "adv_examples_path",
ATTACK_DIR)
main(config_file=parameters_filepath)
create_tex_tables(filespath="./result_files")
print("Check evasion latex table file in results_files")