def main(): # set the path to pre-trained model and output pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth" args.outf = args.outf + args.net_type + "_" + args.dataset + "/" if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == "cifar100": args.num_classes = 100 # load networks if args.net_type == "densenet": if args.dataset == "svhn": model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0), ), ]) elif args.net_type == "resnet": model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print("load model: " + args.net_type) # load dataset print("load target data: ", args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) test_clean_data = torch.load(args.outf + "clean_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_adv_data = torch.load(args.outf + "adv_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_noisy_data = torch.load(args.outf + "noisy_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_label = torch.load(args.outf + "label_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print("get sample mean and covariance") sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print("get LID scores") LID, LID_adv, LID_noisy = lib_generation.get_LID(model, test_clean_data, test_adv_data, test_noisy_data, test_label, num_output) overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90] list_counter = 0 for overlap in overlap_list: Save_LID = np.asarray(LID[list_counter], dtype=np.float32) Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32) Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32) Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy)) LID_data, LID_labels = lib_generation.merge_and_generate_labels( Save_LID_adv, Save_LID_pos) file_name = os.path.join( args.outf, "LID_%s_%s_%s.npy" % (overlap, args.dataset, args.adv_type)) LID_data = np.concatenate((LID_data, LID_labels), axis=1) np.save(file_name, LID_data) list_counter += 1 print("get Mahalanobis scores") m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print("\nNoise: " + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score_adv( model, test_clean_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score_adv( model, test_adv_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy = lib_generation.get_Mahalanobis_score_adv( model, test_noisy_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, "Mahalanobis_%s_%s_%s.npy" % (str(magnitude), args.dataset, args.adv_type), ) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'malaria': args.num_classes = 2 if args.dataset == 'cifar100': args.num_classes = 100 # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) elif args.net_type == 'resnet18': model = models.get_model(args.net_type) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) model.cuda() print('load model: ' + args.net_type) # load dataset # selected_list = torch.LongTensor(selected_list) # test_clean_data = torch.index_select(test_clean_data, 0, selected_list) # test_adv_data = torch.index_select(test_adv_data, 0, selected_list) # test_noisy_data = torch.index_select(test_noisy_data, 0, selected_list) # test_label = torch.index_select(test_label, 0, selected_list) save_path = '%s/%s_%s/' % (args.outf, args.net_type, args.dataset) os.makedirs(save_path, exist_ok=True) # torch.save(test_clean_data, '%s/clean_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_adv_data, '%s/adv_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_noisy_data, '%s/noisy_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_label, '%s/label_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) print('load target data: ', args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, None, args.dataroot) test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get LID scores') LID, LID_adv, LID_noisy \ = lib_generation.get_LID(model, test_clean_data, test_adv_data, test_noisy_data, test_label, num_output) overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90] list_counter = 0 for overlap in overlap_list: Save_LID = np.asarray(LID[list_counter], dtype=np.float32) Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32) Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32) Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy)) LID_data, LID_labels = lib_generation.merge_and_generate_labels( Save_LID_adv, Save_LID_pos) file_name = os.path.join( args.outf, 'LID_%s_%s_%s.npy' % (overlap, args.dataset, args.adv_type)) LID_data = np.concatenate((LID_data, LID_labels), axis=1) np.save(file_name, LID_data) list_counter += 1 print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('\nNoise: ' + str(magnitude)) for i in range(num_output): M_in \ = lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out \ = lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy \ = lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, args.adv_type)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def get_features_for_regressor(regressor_feature, model, config, test_loader, dataset, i, out_flag, device, class_mean, class_precision, tied_precision, pca_list, knn_search_list, knn_mean, knn_precision): if regressor_feature == 'mahalanobis_class_cov': print("Getting scores using Mahalanobis class covoriance") scores = [] for magnitude in config['exp_params']['noise_params']['m_list']: cur_score = lib_generation.get_Mahalanobis_score( regressor_feature, model, config, test_loader, out_flag, class_mean, class_precision, class_mean, class_precision, i, magnitude, knn_search_list[i], device, knn=False) cur_score = np.array(cur_score) scores.append(cur_score.reshape(-1, 1)) return np.hstack(scores) elif regressor_feature == 'mahalanobis_tied_cov': print( "Getting scores using Mahalanobis tied covoriance [Mahalanobis Paper]" ) scores = [] for magnitude in config['exp_params']['noise_params']['m_list']: cur_score = lib_generation.get_Mahalanobis_score( regressor_feature, model, config, test_loader, out_flag, class_mean, tied_precision, class_mean, tied_precision, i, magnitude, knn_search_list[i], device, knn=False) cur_score = np.array(cur_score) scores.append(cur_score.reshape(-1, 1)) return np.hstack(scores) elif regressor_feature == 'pca': print("Getting scores using PCA") scores = [] for magnitude in config['exp_params']['noise_params']['m_list']: cur_score = lib_generation.get_pca_score( model, config, test_loader, out_flag, class_mean, class_precision, i, magnitude, pca_list, device) cur_score = np.array(cur_score) scores.append(cur_score.reshape(-1, 1)) return np.hstack(scores) elif regressor_feature == 'knn_mahalanobis_class_cov': print("Getting scores using Mahalanobis class covoriance on K-NNs") scores = [] for magnitude in config['exp_params']['noise_params']['m_list']: cur_score = lib_generation.get_Mahalanobis_score( regressor_feature, model, config, test_loader, out_flag, class_mean, class_precision, knn_mean, knn_precision, i, magnitude, knn_search_list[i], device, knn=True) cur_score = np.array(cur_score) scores.append(cur_score.reshape(-1, 1)) return np.hstack(scores) elif regressor_feature == 'knn_mahalanobis_tied_cov': print("Getting scores using Mahalanobis tied covoriance on K-NNs") scores = [] for magnitude in config['exp_params']['noise_params']['m_list']: cur_score = lib_generation.get_Mahalanobis_score( regressor_feature, model, config, test_loader, out_flag, class_mean, tied_precision, knn_mean, knn_precision, i, magnitude, knn_search_list[i], device, knn=True) cur_score = np.array(cur_score) scores.append(cur_score.reshape(-1, 1)) return np.hstack(scores) elif regressor_feature == 'ODIN': scores = [] for params in config['exp_params']['odin_args']['settings']: cur_score = lib_generation.get_posterior( model, config['model_params']['net_type'], test_loader, params[1], params[0], config['logging_params']['outf'], out_flag, device) scores.append(cur_score.reshape(-1, 1)) return np.hstack(scores) elif regressor_feature == 'LID': # dumping code os.system( "python ADV_Samples.py --dataset {} --net_type {} --adv_type {} --gpu {} --outf {} --model {} --ood_idx {} --num_oods {}" .format(dataset, config['model_params']['net_type'], config['exp_params']['lid_args']['adv_type'], config['trainer_params']['gpu'], config['exp_params']['lid_args']['outf'], config['exp_params']['dataset'], config['model_params']['out_idx'], config['model_params']['num_oods'])) # scoring code base_path = config['exp_params']['lid_args']['outf'] + config[ 'model_params']['net_type'] + '_' + dataset + '/' test_clean_data = torch.load( base_path + 'clean_data_%s_%s_%s.pth' % (config['model_params']['net_type'], dataset, config['exp_params']['lid_args']['adv_type'])) test_adv_data = torch.load( base_path + 'adv_data_%s_%s_%s.pth' % (config['model_params']['net_type'], dataset, config['exp_params']['lid_args']['adv_type'])) test_noisy_data = torch.load( base_path + 'noisy_data_%s_%s_%s.pth' % (config['model_params']['net_type'], dataset, config['exp_params']['lid_args']['adv_type'])) test_label = torch.load(base_path + 'label_%s_%s_%s.pth' % (config['model_params']['net_type'], dataset, config['exp_params']['lid_args']['adv_type'])) LID, LID_adv, LID_noisy = lib_generation.get_LID( model, test_clean_data, test_adv_data, test_noisy_data, test_label, i + 1) LID_scores = np.hstack([np.vstack(s) for s in LID]) print("LID_scores_shape:", LID_scores.shape) return LID_scores else: raise Exception("Wrong type of regressor feature")