def __init__(self, args): self.args = args self.epoch = None # create dataset image_loaders = loaders.ImageLoader(args) self.trainset_loader_for_train, self.trainset_loader_for_infer, self.valset_loader, self.outlier_loader = image_loaders.get_loaders() print("\nDATASET:", args.dataset) if self.args.loss.split("_")[0] == "softmax": loss_first_part = losses.SoftMaxLossFirstPart loss_second_part = losses.SoftMaxLossSecondPart elif self.args.loss.split("_")[0] == "isomax": loss_first_part = losses.IsoMaxLossFirstPart loss_second_part = losses.IsoMaxLossSecondPart elif self.args.loss.split("_")[0] == "isomaxplus": loss_first_part = losses.IsoMaxPlusLossFirstPart loss_second_part = losses.IsoMaxPlusLossSecondPart else: sys.exit('You should pass a valid loss to use!!!') # create model print("=> creating model '{}'".format(self.args.model_name)) if self.args.model_name == "densenetbc100": self.model = models.DenseNet3(100, int(self.args.number_of_model_classes), loss_first_part=loss_first_part) elif self.args.model_name == "resnet110": self.model = models.ResNet110(num_c=self.args.number_of_model_classes, loss_first_part=loss_first_part) self.model.cuda() # print and save model arch print("\nMODEL:", self.model) with open(os.path.join(self.args.experiment_path, 'model.arch'), 'w') as file: print(self.model, file=file) print("\n$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") utils.print_num_params(self.model) print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\n") # create loss self.criterion = loss_second_part() parameters = self.model.parameters() self.optimizer = torch.optim.SGD( parameters, lr=self.args.original_learning_rate, momentum=self.args.momentum, nesterov=True, weight_decay=args.weight_decay) self.scheduler = torch.optim.lr_scheduler.MultiStepLR( self.optimizer, milestones=self.args.learning_rate_decay_epochs, gamma=args.learning_rate_decay_rate) print("\nTRAIN:", self.criterion, self.optimizer, self.scheduler)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) for i, (name, module) in enumerate(model._modules.items()): module = recursion_change_bn(model) for m in model.modules(): if 'Conv' in str(type(m)): setattr(m, 'padding_mode', 'zeros') in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('\nNoise: ' + str(magnitude)) for i in range(num_output): M_in \ = lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out \ = lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy \ = lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, 'FGSM')) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth" args.outf = args.outf + args.net_type + "_" + args.dataset + "/" if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == "cifar100": args.num_classes = 100 if args.dataset == "svhn": out_dist_list = ["cifar10", "imagenet_resize", "lsun_resize"] else: out_dist_list = ["svhn", "imagenet_resize", "lsun_resize"] # load networks if args.net_type == "densenet": if args.dataset == "svhn": model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0), ), ]) elif args.net_type == "resnet": model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print("load model: " + args.net_type) # load dataset print("load target data: ", args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print("get sample mean and covariance") sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print("get Mahalanobis scores") m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print("Noise: " + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score( model, test_loader, args.num_classes, args.outf, True, args.net_type, sample_mean, precision, i, magnitude, ) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print("Out-distribution: " + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score( model, out_test_loader, args.num_classes, args.outf, False, args.net_type, sample_mean, precision, i, magnitude, ) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) ( Mahalanobis_data, Mahalanobis_labels, ) = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, "Mahalanobis_%s_%s_%s.npy" % (str(magnitude), args.dataset, out_dist), ) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 if args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict(torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict(torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get Mahalanobis scores') # m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] m_list = [0.0] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): if args.train_flag == False: print("OOD: True") lib_generation.get_activations(model, test_loader, args.outf, i) if args.train_flag == True: print("OOD: False") print("Extracting activations for Layer: " + str(i)) lib_generation.get_activations(model, train_loader, args.outf, i)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'malaria': args.num_classes = 2 if args.dataset == 'cifar100': args.num_classes = 100 # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) elif args.net_type == 'resnet18': model = models.get_model(args.net_type) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) model.cuda() print('load model: ' + args.net_type) # load dataset # selected_list = torch.LongTensor(selected_list) # test_clean_data = torch.index_select(test_clean_data, 0, selected_list) # test_adv_data = torch.index_select(test_adv_data, 0, selected_list) # test_noisy_data = torch.index_select(test_noisy_data, 0, selected_list) # test_label = torch.index_select(test_label, 0, selected_list) save_path = '%s/%s_%s/' % (args.outf, args.net_type, args.dataset) os.makedirs(save_path, exist_ok=True) # torch.save(test_clean_data, '%s/clean_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_adv_data, '%s/adv_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_noisy_data, '%s/noisy_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_label, '%s/label_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) print('load target data: ', args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, None, args.dataroot) test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get LID scores') LID, LID_adv, LID_noisy \ = lib_generation.get_LID(model, test_clean_data, test_adv_data, test_noisy_data, test_label, num_output) overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90] list_counter = 0 for overlap in overlap_list: Save_LID = np.asarray(LID[list_counter], dtype=np.float32) Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32) Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32) Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy)) LID_data, LID_labels = lib_generation.merge_and_generate_labels( Save_LID_adv, Save_LID_pos) file_name = os.path.join( args.outf, 'LID_%s_%s_%s.npy' % (overlap, args.dataset, args.adv_type)) LID_data = np.concatenate((LID_data, LID_labels), axis=1) np.save(file_name, LID_data) list_counter += 1 print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('\nNoise: ' + str(magnitude)) for i in range(num_output): M_in \ = lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out \ = lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy \ = lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, args.adv_type)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): dir_path = os.path.join("experiments", args.dir, "train_classify", "data~"+args.dataset+"+model~"+args.net_type+"+loss~"+str(args.loss)) file_path = os.path.join(dir_path, "results_odd.csv") with open(file_path, "w") as results_file: results_file.write( "EXECUTION,MODEL,IN-DATA,OUT-DATA,LOSS,AD-HOC,SCORE,INFER-LEARN,INFER-TRANS," "TNR,AUROC,DTACC,AUIN,AUOUT,CPU_FALSE,CPU_TRUE,GPU_FALSE,GPU_TRUE,TEMPERATURE,MAGNITUDE\n") args_outf = os.path.join("temporary", args.dir, args.loss, args.net_type + '+' + args.dataset) if os.path.isdir(args_outf) == False: os.makedirs(args_outf) # define number of classes if args.dataset == 'cifar100': args.num_classes = 100 elif args.dataset == 'imagenet32': args.num_classes = 1000 else: args.num_classes = 10 if args.dataset == 'cifar10': out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] elif args.dataset == 'cifar100': out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] elif args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] if args.dataset == 'cifar10': in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.491, 0.482, 0.446), (0.247, 0.243, 0.261))]) elif args.dataset == 'cifar100': in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.507, 0.486, 0.440), (0.267, 0.256, 0.276))]) elif args.dataset == 'svhn': in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.437, 0.443, 0.472), (0.198, 0.201, 0.197))]) for args.execution in range(1, args.executions + 1): print("EXECUTION:", args.execution) pre_trained_net = os.path.join(dir_path, "model" + str(args.execution) + ".pth") if args.loss.split("_")[0] == "softmax": loss_first_part = losses.SoftMaxLossFirstPart scores = ["ES"] elif args.loss.split("_")[0] == "isomax": loss_first_part = losses.IsoMaxLossFirstPart scores = ["ES"] elif args.loss.split("_")[0] == "isomaxplus": loss_first_part = losses.IsoMaxPlusLossFirstPart scores = ["MDS"] # load networks if args.net_type == 'densenetbc100': model = models.DenseNet3(100, int(args.num_classes), loss_first_part=loss_first_part) elif args.net_type == 'resnet110': model = models.ResNet110(num_c=args.num_classes, loss_first_part=loss_first_part) model.load_state_dict(torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target valid data: ', args.dataset) _, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) for score in scores: print("\n\n\n###############################") print("###############################") print("SCORE:", score) print("###############################") print("###############################") base_line_list = [] get_scores(model, test_loader, args_outf, True, score) out_count = 0 for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet(out_dist, args.batch_size, in_transform, args.dataroot) print('Out-distribution: ' + out_dist) get_scores(model, out_test_loader, args_outf, False, score) test_results = callog.metric(args_outf, ['PoT']) base_line_list.append(test_results) out_count += 1 # print the results mtypes = ['TNR', 'AUROC', 'DTACC', 'AUIN', 'AUOUT'] print('Baseline method: train in_distribution: ' + args.dataset + '==========') count_out = 0 for results in base_line_list: print('out_distribution: '+ out_dist_list[count_out]) for mtype in mtypes: print(' {mtype:6s}'.format(mtype=mtype), end='') print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='') print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='') print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='') print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='') print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='') print('') #Saving odd results: with open(file_path, "a") as results_file: results_file.write("{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}\n".format( str(args.execution), args.net_type, args.dataset, out_dist_list[count_out], str(args.loss), "NATIVE", score, 'NO', False, '{:.2f}'.format(100.*results['PoT']['TNR']), '{:.2f}'.format(100.*results['PoT']['AUROC']), '{:.2f}'.format(100.*results['PoT']['DTACC']), '{:.2f}'.format(100.*results['PoT']['AUIN']), '{:.2f}'.format(100.*results['PoT']['AUOUT']), 0, 0, 0, 0, 1, 0)) count_out += 1
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 if args.adv_type == 'FGSM': adv_noise = 0.05 elif args.adv_type == 'BIM': adv_noise = 0.01 elif args.adv_type == 'DeepFool': if args.net_type == 'resnet': if args.dataset == 'cifar10': adv_noise = 0.18 elif args.dataset == 'cifar100': adv_noise = 0.03 else: adv_noise = 0.1 else: if args.dataset == 'cifar10': adv_noise = 0.6 elif args.dataset == 'cifar100': adv_noise = 0.1 else: adv_noise = 0.5 # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([transforms.ToTensor(), \ transforms.Normalize((125.3/255, 123.0/255, 113.9/255), \ (63.0/255, 62.1/255.0, 66.7/255.0)),]) min_pixel = -1.98888885975 max_pixel = 2.12560367584 if args.dataset == 'cifar10': if args.adv_type == 'FGSM': random_noise_size = 0.21 / 4 elif args.adv_type == 'BIM': random_noise_size = 0.21 / 4 elif args.adv_type == 'DeepFool': random_noise_size = 0.13 * 2 / 10 elif args.adv_type == 'CWL2': random_noise_size = 0.03 / 2 elif args.dataset == 'cifar100': if args.adv_type == 'FGSM': random_noise_size = 0.21 / 8 elif args.adv_type == 'BIM': random_noise_size = 0.21 / 8 elif args.adv_type == 'DeepFool': random_noise_size = 0.13 * 2 / 8 elif args.adv_type == 'CWL2': random_noise_size = 0.06 / 5 else: if args.adv_type == 'FGSM': random_noise_size = 0.21 / 4 elif args.adv_type == 'BIM': random_noise_size = 0.21 / 4 elif args.adv_type == 'DeepFool': random_noise_size = 0.16 * 2 / 5 elif args.adv_type == 'CWL2': random_noise_size = 0.07 / 2 elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([transforms.ToTensor(), \ transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),]) min_pixel = -2.42906570435 max_pixel = 2.75373125076 if args.dataset == 'cifar10': if args.adv_type == 'FGSM': random_noise_size = 0.25 / 4 elif args.adv_type == 'BIM': random_noise_size = 0.13 / 2 elif args.adv_type == 'DeepFool': random_noise_size = 0.25 / 4 elif args.adv_type == 'CWL2': random_noise_size = 0.05 / 2 elif args.dataset == 'cifar100': if args.adv_type == 'FGSM': random_noise_size = 0.25 / 8 elif args.adv_type == 'BIM': random_noise_size = 0.13 / 4 elif args.adv_type == 'DeepFool': random_noise_size = 0.13 / 4 elif args.adv_type == 'CWL2': random_noise_size = 0.05 / 2 else: if args.adv_type == 'FGSM': random_noise_size = 0.25 / 4 elif args.adv_type == 'BIM': random_noise_size = 0.13 / 2 elif args.adv_type == 'DeepFool': random_noise_size = 0.126 elif args.adv_type == 'CWL2': random_noise_size = 0.05 / 1 model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) _, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) print('Attack: ' + args.adv_type + ', Dist: ' + args.dataset + '\n') model.eval() adv_data_tot, clean_data_tot, noisy_data_tot = 0, 0, 0 label_tot = 0 correct, adv_correct, noise_correct = 0, 0, 0 total, generated_noise = 0, 0 criterion = nn.CrossEntropyLoss().cuda() selected_list = [] selected_index = 0 for data, target in test_loader: data, target = data.cuda(), target.cuda() data, target = Variable(data, volatile=True), Variable(target) output = model(data) # compute the accuracy pred = output.data.max(1)[1] equal_flag = pred.eq(target.data).cpu() correct += equal_flag.sum() noisy_data = torch.add(data.data, random_noise_size, torch.randn(data.size()).cuda()) noisy_data = torch.clamp(noisy_data, min_pixel, max_pixel) if total == 0: clean_data_tot = data.clone().data.cpu() label_tot = target.clone().data.cpu() noisy_data_tot = noisy_data.clone().cpu() else: clean_data_tot = torch.cat( (clean_data_tot, data.clone().data.cpu()), 0) label_tot = torch.cat((label_tot, target.clone().data.cpu()), 0) noisy_data_tot = torch.cat( (noisy_data_tot, noisy_data.clone().cpu()), 0) # generate adversarial model.zero_grad() inputs = Variable(data.data, requires_grad=True) output = model(inputs) loss = criterion(output, target) loss.backward() if args.adv_type == 'FGSM': gradient = torch.ge(inputs.grad.data, 0) gradient = (gradient.float() - 0.5) * 2 if args.net_type == 'densenet': gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \ gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0/255.0)) gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \ gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1/255.0)) gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \ gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7/255.0)) else: gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \ gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023)) gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \ gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994)) gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \ gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010)) elif args.adv_type == 'BIM': gradient = torch.sign(inputs.grad.data) for k in range(5): inputs = torch.add(inputs.data, adv_noise, gradient) inputs = torch.clamp(inputs, min_pixel, max_pixel) inputs = Variable(inputs, requires_grad=True) output = model(inputs) loss = criterion(output, target) loss.backward() gradient = torch.sign(inputs.grad.data) if args.net_type == 'densenet': gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \ gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0/255.0)) gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \ gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1/255.0)) gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \ gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7/255.0)) else: gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \ gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023)) gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \ gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994)) gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \ gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010)) if args.adv_type == 'DeepFool': _, adv_data = adversary.deepfool(model, data.data.clone(), target.data.cpu(), \ args.num_classes, step_size=adv_noise, train_mode=False) adv_data = adv_data.cuda() elif args.adv_type == 'CWL2': _, adv_data = adversary.cw(model, data.data.clone(), target.data.cpu(), 1.0, 'l2', crop_frac=1.0) else: adv_data = torch.add(inputs.data, adv_noise, gradient) adv_data = torch.clamp(adv_data, min_pixel, max_pixel) # measure the noise temp_noise_max = torch.abs( (data.data - adv_data).view(adv_data.size(0), -1)) temp_noise_max, _ = torch.max(temp_noise_max, dim=1) generated_noise += torch.sum(temp_noise_max) if total == 0: flag = 1 adv_data_tot = adv_data.clone().cpu() else: adv_data_tot = torch.cat((adv_data_tot, adv_data.clone().cpu()), 0) output = model(Variable(adv_data, volatile=True)) # compute the accuracy pred = output.data.max(1)[1] equal_flag_adv = pred.eq(target.data).cpu() adv_correct += equal_flag_adv.sum() output = model(Variable(noisy_data, volatile=True)) # compute the accuracy pred = output.data.max(1)[1] equal_flag_noise = pred.eq(target.data).cpu() noise_correct += equal_flag_noise.sum() for i in range(data.size(0)): if equal_flag[i] == 1 and equal_flag_noise[ i] == 1 and equal_flag_adv[i] == 0: selected_list.append(selected_index) selected_index += 1 total += data.size(0) selected_list = torch.LongTensor(selected_list) clean_data_tot = torch.index_select(clean_data_tot, 0, selected_list) adv_data_tot = torch.index_select(adv_data_tot, 0, selected_list) noisy_data_tot = torch.index_select(noisy_data_tot, 0, selected_list) label_tot = torch.index_select(label_tot, 0, selected_list) torch.save( clean_data_tot, '%s/clean_data_%s_%s_%s.pth' % (args.outf, args.net_type, args.dataset, args.adv_type)) torch.save( adv_data_tot, '%s/adv_data_%s_%s_%s.pth' % (args.outf, args.net_type, args.dataset, args.adv_type)) torch.save( noisy_data_tot, '%s/noisy_data_%s_%s_%s.pth' % (args.outf, args.net_type, args.dataset, args.adv_type)) torch.save( label_tot, '%s/label_%s_%s_%s.pth' % (args.outf, args.net_type, args.dataset, args.adv_type)) print('Adversarial Noise:({:.2f})\n'.format(generated_noise / total)) print('Final Accuracy: {}/{} ({:.2f}%)\n'.format(correct, total, 100. * correct / total)) print('Adversarial Accuracy: {}/{} ({:.2f}%)\n'.format( adv_correct, total, 100. * adv_correct / total)) print('Noisy Accuracy: {}/{} ({:.2f}%)\n'.format( noise_correct, total, 100. * noise_correct / total))
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 if args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 ################################ edits print("Calculate SVD before getting sample mean and variance") # svd_result= lib_generation.get_pca(model, args.num_classes, feature_list, train_loader) # lib_generation.get_pca_incremental(model, args.num_classes, feature_list, train_loader,args) svd_result = None print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader, svd_result, args) ################################ edits_end_sample_generator print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \ True, args.net_type, sample_mean, precision, i, magnitude,svd_result,args) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print('Out-distribution: ' + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \ False, args.net_type, sample_mean, precision, i, magnitude,svd_result,args) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, out_dist)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 # load networks if args.net_type == 'densenet': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) test_clean_data, test_adv_data, test_noisy_data, test_label = {}, {}, {}, {} clean_loaders, adv_loaders, noisy_loaders = {}, {}, {} adv_types = ['FGSM', 'BIM', 'DeepFool', 'CWL2', 'PGD100'] for adv_type in adv_types: test_clean_data[adv_type] = torch.load( args.outf + 'clean_data_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) test_adv_data[adv_type] = torch.load( args.outf + 'adv_data_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) test_noisy_data[adv_type] = torch.load( args.outf + 'noisy_data_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) test_label[adv_type] = torch.load( args.outf + 'label_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) clean_loaders[adv_type] = torch.utils.data.DataLoader( test_clean_data[adv_type], batch_size=args.batch_size) adv_loaders[adv_type] = torch.utils.data.DataLoader( test_adv_data[adv_type], batch_size=args.batch_size) noisy_loaders[adv_type] = torch.utils.data.DataLoader( test_noisy_data[adv_type], batch_size=args.batch_size) # nasty hack train_clean_x = [] train_clean_y = [] for X, y in tqdm(train_loader, desc='Aggregating train dataset', dynamic_ncols=True): train_clean_x.append(X.cpu().numpy()) train_clean_y.append(X.cpu().numpy()) train_clean_x = np.concatenate(train_clean_x, axis=0) train_clean_y = np.concatenate(train_clean_y, axis=0) X, y = train_clean_x, train_clean_y model.eval() # datasets = {} method_name = 'odd_odds' dir_name = f'{method_name}_{args.net_type}_{args.dataset}' os.makedirs(dir_name, exist_ok=True) # "known" variant # results_filename = f'{dir_name}/results_known.txt' # with open(results_filename, 'w') as results_file: # for adv_type in adv_types: # train_split = 0.1 # train_size = int(train_split * len(test_clean_data[adv_type])) # test_size = len(test_clean_data[adv_type]) - train_size # X = np.concatenate([ # test_clean_data[adv_type][:train_size], # test_adv_data[adv_type][:train_size], # test_noisy_data[adv_type][:train_size], # ]) # label_y = np.concatenate([ # test_label[adv_type][:train_size], # test_label[adv_type][:train_size], # test_label[adv_type][:train_size], # ]) # adv_y = np.concatenate([ # np.ones(train_size), # np.zeros(train_size), # np.ones(train_size), # ]) # X_test = np.concatenate([ # test_clean_data[adv_type][train_size:], # test_adv_data[adv_type][train_size:], # test_noisy_data[adv_type][train_size:], # ]) # label_y_test = np.concatenate([ # test_label[adv_type][train_size:], # test_label[adv_type][train_size:], # test_label[adv_type][train_size:], # ]) # adv_y_test = np.concatenate([ # np.ones(test_size), # np.zeros(test_size), # np.ones(test_size), # ]) # "unsupervised" variant results_filename = f'{dir_name}/results_unsupervised.txt' with open(results_filename, 'w') as results_file: for adv_type in adv_types: test_size = len(test_clean_data[adv_type]) X_test = np.concatenate([ test_clean_data[adv_type], test_adv_data[adv_type], test_noisy_data[adv_type], ]) label_y_test = np.concatenate([ test_label[adv_type], test_label[adv_type], test_label[adv_type], ]) adv_y_test = np.concatenate([ np.ones(test_size), np.zeros(test_size), np.ones(test_size), ]) adv_y_test_switched = np.concatenate([ np.zeros(test_size), np.ones(test_size), np.zeros(test_size), ]) # "train" class_vectors = model.get_class_vectors() num_classes = class_vectors.size(0) # parameters taken from tensorflow file: # noise_eps = 'n18.0,n24.0,n30.0'.split(',') # noise_eps_detect = 'n30.0'.split(',') # parameters taken from pytorch file: noise_eps = 'n0.01,s0.01,u0.01,n0.02,s0.02,u0.02,s0.03,n0.03,u0.03'.split( ',') noise_eps_detect = 'n0.003,s0.003,u0.003,n0.005,s0.005,u0.005,s0.008,n0.008,u0.008'.split( ',') # noise_eps_detect = None clip_min = min(X.min(), X_test.min()) clip_max = max(X.max(), X_test.max()) predictor = collect_statistics( X, y, # these are class labels, not adversarial binary labels latent_and_logits_fn=model.forward_with_latent, nb_classes=args.num_classes, weights=model.get_class_vectors(), targeted=False, noise_eps=noise_eps, noise_eps_detect=noise_eps_detect, num_noise_samples=256, batch_size=128, clip_min=clip_min, clip_max=clip_max, p_ratio_cutoff=0.999, clip_alignments=True, cache_alignments_dir=dir_name, ) next(predictor) # test y_pred_list = [] y_score_list = [] batch_size = 128 batches, remainder = divmod(X_test.shape[0], batch_size) if remainder > 0: batches += 1 for i in tqdm(range(batches), dynamic_ncols=True, desc=f"evaluating on {adv_type}"): X_test_batch = X_test[i * batch_size:(i + 1) * batch_size] corrected_batch, y_pred_batch, decision_batch = predictor.send( X_test_batch).T # print(f'corrected_batch.shape: {corrected_batch.shape}') # print(f'y_pred_batch.shape: {y_pred_batch.shape}') # print(f'y_pred_batch: {y_pred_batch}') y_pred_list.append(y_pred_batch) y_score_list.append(decision_batch) y_pred = np.concatenate(y_pred_list, axis=0) y_scores = np.concatenate(y_score_list, axis=0) acc = accuracy_score(adv_y_test_switched, y_pred) auroc = roc_auc_score(adv_y_test_switched, y_scores) print(f'Accuracy on {adv_type}: {acc}', file=results_file) print(f'AUROC on {adv_type}: {auroc}', file=results_file) acc = accuracy_score(adv_y_test, y_pred) auroc = roc_auc_score(adv_y_test, y_scores) print(f'Accuracy (original labels) on {adv_type}: {acc}', file=results_file) print(f'AUROC (original labels) on {adv_type}: {auroc}', file=results_file)
def main(): # set the path to pre-trained model and output pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth" args.outf = args.outf + args.net_type + "_" + args.dataset + "/" if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == "cifar100": args.num_classes = 100 if args.dataset == "svhn": out_dist_list = ["cifar10", "imagenet_resize", "lsun_resize"] else: out_dist_list = ["svhn", "imagenet_resize", "lsun_resize"] # load networks if args.net_type == "densenet": if args.dataset == "svhn": model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) ) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose( [ transforms.ToTensor(), transforms.Normalize( (125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0), ), ] ) elif args.net_type == "resnet": model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) ) in_transform = transforms.Compose( [ transforms.ToTensor(), transforms.Normalize( (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010) ), ] ) model.cuda() print("load model: " + args.net_type) # load dataset print("load target data: ", args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot ) # measure the performance M_list = [0, 0.0005, 0.001, 0.0014, 0.002, 0.0024, 0.005, 0.01, 0.05, 0.1, 0.2] T_list = [1, 10, 100, 1000] base_line_list = [] ODIN_best_tnr = [0, 0, 0] ODIN_best_results = [0, 0, 0] ODIN_best_temperature = [-1, -1, -1] ODIN_best_magnitude = [-1, -1, -1] for T in T_list: for m in M_list: magnitude = m temperature = T lib_generation.get_posterior( model, args.net_type, test_loader, magnitude, temperature, args.outf, True, ) out_count = 0 print("Temperature: " + str(temperature) + " / noise: " + str(magnitude)) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot ) print("Out-distribution: " + out_dist) lib_generation.get_posterior( model, args.net_type, out_test_loader, magnitude, temperature, args.outf, False, ) if temperature == 1 and magnitude == 0: test_results = callog.metric(args.outf, ["PoT"]) base_line_list.append(test_results) else: val_results = callog.metric(args.outf, ["PoV"]) if ODIN_best_tnr[out_count] < val_results["PoV"]["TNR"]: ODIN_best_tnr[out_count] = val_results["PoV"]["TNR"] ODIN_best_results[out_count] = callog.metric(args.outf, ["PoT"]) ODIN_best_temperature[out_count] = temperature ODIN_best_magnitude[out_count] = magnitude out_count += 1 # print the results mtypes = ["TNR", "AUROC", "DTACC", "AUIN", "AUOUT"] print("Baseline method: in_distribution: " + args.dataset + "==========") count_out = 0 for results in base_line_list: print("out_distribution: " + out_dist_list[count_out]) for mtype in mtypes: print(" {mtype:6s}".format(mtype=mtype), end="") print("\n{val:6.2f}".format(val=100.0 * results["PoT"]["TNR"]), end="") print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUROC"]), end="") print(" {val:6.2f}".format(val=100.0 * results["PoT"]["DTACC"]), end="") print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUIN"]), end="") print(" {val:6.2f}\n".format(val=100.0 * results["PoT"]["AUOUT"]), end="") print("") count_out += 1 print("ODIN method: in_distribution: " + args.dataset + "==========") count_out = 0 for results in ODIN_best_results: print("out_distribution: " + out_dist_list[count_out]) for mtype in mtypes: print(" {mtype:6s}".format(mtype=mtype), end="") print("\n{val:6.2f}".format(val=100.0 * results["PoT"]["TNR"]), end="") print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUROC"]), end="") print(" {val:6.2f}".format(val=100.0 * results["PoT"]["DTACC"]), end="") print(" {val:6.2f}".format(val=100.0 * results["PoT"]["AUIN"]), end="") print(" {val:6.2f}\n".format(val=100.0 * results["PoT"]["AUOUT"]), end="") print("temperature: " + str(ODIN_best_temperature[count_out])) print("magnitude: " + str(ODIN_best_magnitude[count_out])) print("") count_out += 1
def __init__(self, args): self.args = args self.epoch = None self.cluster_predictions_transformation = [] # create dataset if self.args.data_type == 'image': image_loaders = loaders.ImageLoader(args) (self.trainset_first_partition_loader_for_train, self.trainset_second_partition_loader_for_train, self.trainset_first_partition_loader_for_infer, self.trainset_second_partition_loader_for_infer, self.valset_loader, self.normalize) = image_loaders.get_loaders() self.batch_normalize = loaders.BatchNormalize( self.normalize.mean, self.normalize.std, inplace=True, device=torch.cuda.current_device()) elif self.args.data_type == 'text': text_loaders = loaders.TextLoader(args) (self.trainset_first_partition_loader_for_train, self.trainset_second_partition_loader_for_train, self.trainset_first_partition_loader_for_infer, self.trainset_second_partition_loader_for_infer, self.valset_loader, self.normalize) = text_loaders.get_loaders() if self.args.partition == "1": self.trainset_loader_for_train = self.trainset_first_partition_loader_for_train elif self.args.partition == "2": self.trainset_loader_for_train = self.trainset_second_partition_loader_for_train print("\nDATASET:", args.dataset_full) # create model torch.manual_seed(self.args.execution_seed) torch.cuda.manual_seed(self.args.execution_seed) print("=> creating model '{}'".format(self.args.model_name)) if self.args.model_name == "densenetbc100": self.model = models.DenseNet3( 100, int(self.args.number_of_model_classes)) elif self.args.model_name == "resnet32": self.model = models.ResNet32( num_c=self.args.number_of_model_classes) elif self.args.model_name == "resnet34": self.model = models.ResNet34( num_c=self.args.number_of_model_classes) elif self.args.model_name == "resnet110": self.model = models.ResNet110( num_c=self.args.number_of_model_classes) elif self.args.model_name == "resnet50": self.model = models.ResNet50( num_c=self.args.number_of_model_classes) elif self.args.model_name == "resnet18_": self.model = models.resnet18_( num_classes=self.args.number_of_model_classes) elif self.args.model_name == "resnet50_": self.model = models.resnet18_( num_classes=self.args.number_of_model_classes) elif self.args.model_name == "resnet34_": self.model = models.resnet34_( num_classes=self.args.number_of_model_classes) elif self.args.model_name == "resnet101_": self.model = models.resnet101_( num_classes=self.args.number_of_model_classes) elif self.args.model_name == "wideresnet3410": self.model = models.Wide_ResNet( depth=34, widen_factor=10, num_classes=self.args.number_of_model_classes) #elif self.args.model_name == "vgg": # self.model = models.VGG19(num_classes=self.args.number_of_model_classes) #elif self.args.model_name == "efficientnetb0_": # self.model = models.EfficientNet_( # num_classes=self.args.number_of_model_classes, width_coefficient=1.0, depth_coefficient=1.0, dropout_rate=0.2) elif self.args.model_name == "efficientnetb0": self.model = models.EfficientNetB0( num_classes=self.args.number_of_model_classes) elif self.args.model_name == "mobilenetv3": self.model = models.MobileNetV3( mode='large', classes_num=self.args.number_of_model_classes, input_size=32, width_multiplier=1) ############################################################################################################################# ############################################################################################################################# elif self.args.model_name == "textrnn": self.model = models.TextRNN( #self.args.text_config,len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings) len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes) ############################################################## #elif self.args.model_name == "textcnn": # self.model = models.TextCNN( # len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes) #elif self.args.model_name == "textcnn1d": # self.model = models.TextCNN1D( # len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes) #elif self.args.model_name == "attbilstm": # self.model = models.AttBiLSTM( # len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes) elif self.args.model_name == "rcnn": self.model = models.RCNN( #self.args.text_config,len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings) len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes) elif self.args.model_name == "s2satt": self.model = models.Seq2SeqAttention( #self.args.text_config,len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings) len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings, self.args.number_of_model_classes) #elif self.args.model_name == "fasttext": # embed_dim = 64 # self.model = models.FastText( # #self.args.text_config,len(self.args.text_dataset.vocab), self.args.text_dataset.word_embeddings) # #len(train_dataset.get_vocab()), embed_dim, len(train_dataset.get_labels())) # len(text_loaders.train_set.get_vocab()), embed_dim, len(text_loaders.train_set.get_labels())) self.model.cuda() torch.manual_seed(self.args.base_seed) torch.cuda.manual_seed(self.args.base_seed) # print and save model arch... #if self.args.exp_type == "cnn_train": # recent change!!! recent change!!! print("\nMODEL:", self.model) with open(os.path.join(self.args.experiment_path, 'model.arch'), 'w') as file: print(self.model, file=file) print("\n$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") #collect_batch = next(iter(self.trainset_loader_for_train)) #if self.args.data_type == "image": # input_data = collect_batch[0][0]#.cuda() #elif self.args.data_type == "text": # input_data = collect_batch[0].text#.cuda() #summary(self.model, input_data=input_data[0]) utils.print_num_params(self.model) print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") # create loss #self.criterion = losses.GenericLossSecondPart(self.model.classifier).cuda() self.criterion = nn.CrossEntropyLoss().cuda() #total_iteractions = self.args.epochs * self.args.iteractions_per_epoch # create train #print(len(self.trainset_loader_for_train)) if self.args.optim.startswith("sgd"): print("\n$$$$$$$$$$$$$$$") print("OPTIMIZER: SGD") print("$$$$$$$$$$$$$$$\n") initial_learning_rate = float(self.args.optim.split("_")[1][1:]) self.args.epochs = int(self.args.optim.split("_")[2][1:]) weight_decay = float(self.args.optim.split("_")[3][1:]) momentum = float(self.args.optim.split("_")[4][1:]) nesterov = True if (self.args.optim.split("_")[5][1:] == "t") else False #alpha = float(self.args.optim.split("_")[6][1:]) #beta = float(self.args.optim.split("_")[7][1:]) parameters = self.model.parameters() self.optimizer = torch.optim.SGD( parameters, lr=initial_learning_rate, weight_decay=weight_decay, momentum=momentum, nesterov=nesterov) print("INITIAL LEARNING RATE: ", initial_learning_rate) print("TOTAL EPOCHS: ", self.args.epochs) print("WEIGHT DECAY: ", weight_decay) print("MOMENTUM: ", momentum) print("NESTEROV: ", nesterov) #print("ALPHA: ", alpha) #print("BETA: ", beta) elif self.args.optim.startswith("sig"): print("\n$$$$$$$$$$$$$$$") print("OPTIMIZER: SIG") print("$$$$$$$$$$$$$$$\n") initial_learning_rate = float(self.args.optim.split("_")[1][1:]) self.args.epochs = int(self.args.optim.split("_")[2][1:]) weight_decay = float(self.args.optim.split("_")[3][1:]) momentum = float(self.args.optim.split("_")[4][1:]) nesterov = True if (self.args.optim.split("_")[5][1:] == "t") else False alpha = float(self.args.optim.split("_")[6][1:]) beta = float(self.args.optim.split("_")[7][1:]) parameters = self.model.parameters() self.optimizer = torch.optim.SGD( parameters, lr=initial_learning_rate, weight_decay=weight_decay, momentum=momentum, nesterov=nesterov) total_iteractions = self.args.epochs * len(self.trainset_loader_for_train) sig_function = lambda iteraction: 1/(1+math.exp(alpha*(((iteraction+1)/total_iteractions)-beta))) self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda=sig_function, verbose=True) print("INITIAL LEARNING RATE: ", initial_learning_rate) print("TOTAL EPOCHS: ", self.args.epochs) print("WEIGHT DECAY: ", weight_decay) print("MOMENTUM: ", momentum) print("NESTEROV: ", nesterov) print("ALPHA: ", alpha) print("BETA: ", beta) elif self.args.optim.startswith("htd"): print("\n$$$$$$$$$$$$$$$") print("OPTIMIZER: HTD") print("$$$$$$$$$$$$$$$\n") initial_learning_rate = float(self.args.optim.split("_")[1][1:]) self.args.epochs = int(self.args.optim.split("_")[2][1:]) weight_decay = float(self.args.optim.split("_")[3][1:]) momentum = float(self.args.optim.split("_")[4][1:]) nesterov = True if (self.args.optim.split("_")[5][1:] == "t") else False l = float(self.args.optim.split("_")[6][1:]) u = float(self.args.optim.split("_")[7][1:]) parameters = self.model.parameters() self.optimizer = torch.optim.SGD( parameters, lr=initial_learning_rate, weight_decay=weight_decay, momentum=momentum, nesterov=nesterov) #htd_function = lambda epoch: (1-math.tanh(l+(u-l)*((epoch+1)/self.args.epochs)))/2 total_iteractions = self.args.epochs * len(self.trainset_loader_for_train) htd_function = lambda iteraction: (1-math.tanh(l+(u-l)*((iteraction+1)/total_iteractions)))/2 self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda=htd_function, verbose=True) print("INITIAL LEARNING RATE: ", initial_learning_rate) print("TOTAL EPOCHS: ", self.args.epochs) print("WEIGHT DECAY: ", weight_decay) print("MOMENTUM: ", momentum) print("NESTEROV: ", nesterov) print("L: ", l) print("U: ", u) elif self.args.optim.startswith("cos"): print("\n$$$$$$$$$$$$$$$") print("OPTIMIZER: COS") print("$$$$$$$$$$$$$$$\n") initial_learning_rate = float(self.args.optim.split("_")[1][1:]) self.args.epochs = int(self.args.optim.split("_")[2][1:]) weight_decay = float(self.args.optim.split("_")[3][1:]) momentum = float(self.args.optim.split("_")[4][1:]) nesterov = True if (self.args.optim.split("_")[5][1:] == "t") else False #l = float(self.args.optim.split("_")[6][1:]) #u = float(self.args.optim.split("_")[7][1:]) parameters = self.model.parameters() self.optimizer = torch.optim.SGD( parameters, lr=initial_learning_rate, weight_decay=weight_decay, momentum=momentum, nesterov=nesterov) #cos_function = lambda epoch: (1+math.cos(math.pi*((epoch+1)/self.args.epochs)))/2 total_iteractions = self.args.epochs * len(self.trainset_loader_for_train) cos_function = lambda iteraction: (1+math.cos(math.pi*((iteraction+1)/total_iteractions)))/2 self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda=cos_function, verbose=True) print("INITIAL LEARNING RATE: ", initial_learning_rate) print("TOTAL EPOCHS: ", self.args.epochs) print("WEIGHT DECAY: ", weight_decay) print("MOMENTUM: ", momentum) print("NESTEROV: ", nesterov) #print("L: ", l) #print("U: ", u) elif self.args.optim.startswith("adam"): print("\n$$$$$$$$$$$$$$$") print("OPTIMIZER: ADAM") print("$$$$$$$$$$$$$$$\n") initial_learning_rate = float(self.args.optim.split("_")[1][1:]) self.args.epochs = int(self.args.optim.split("_")[2][1:]) weight_decay = float(self.args.optim.split("_")[3][1:]) amsgrad = True if (self.args.optim.split("_")[4][1:] == "t") else False beta_first = float(self.args.optim.split("_")[5][2:]) beta_second = float(self.args.optim.split("_")[6][2:]) parameters = self.model.parameters() self.optimizer = torch.optim.Adam( parameters, lr=initial_learning_rate, amsgrad=amsgrad, weight_decay=weight_decay, betas=(beta_first, beta_second)) print("INITIAL LEARNING RATE: ", initial_learning_rate) print("TOTAL EPOCHS: ", self.args.epochs) print("WEIGHT DECAY: ", weight_decay) print("AMSGRAD: ", amsgrad) print("BETA_FIRST: ", beta_first) print("BEST_SECOND: ", beta_second) elif self.args.optim.startswith("rmsprop"): print("\n$$$$$$$$$$$$$$$$$$") print("OPTIMIZER: RMSPROP") print("$$$$$$$$$$$$$$$$$$\n") initial_learning_rate = float(self.args.optim.split("_")[1][1:]) self.args.epochs = int(self.args.optim.split("_")[2][1:]) weight_decay = float(self.args.optim.split("_")[3][1:]) momentum = float(self.args.optim.split("_")[4][1:]) centered = True if (self.args.optim.split("_")[5][1:] == "t") else False alpha = float(self.args.optim.split("_")[6][1:]) parameters = self.model.parameters() self.optimizer = torch.optim.RMSprop( parameters, lr=initial_learning_rate, weight_decay=weight_decay, momentum=momentum, centered=centered, alpha=alpha) print("INITIAL LEARNING RATE: ", initial_learning_rate) print("TOTAL EPOCHS: ", self.args.epochs) print("WEIGHT DECAY: ", weight_decay) print("MOMENTUM: ", momentum) print("ALPHA: ", alpha) elif self.args.optim.startswith("adagrad"): print("\n$$$$$$$$$$$$$$$$$$") print("OPTIMIZER: ADAGRAD") print("$$$$$$$$$$$$$$$$$$\n") initial_learning_rate = float(self.args.optim.split("_")[1][1:]) self.args.epochs = int(self.args.optim.split("_")[2][1:]) weight_decay = float(self.args.optim.split("_")[3][1:]) lr_decay = float(self.args.optim.split("_")[4][1:]) initial_accumulator_value = float(self.args.optim.split("_")[5][1:]) parameters = self.model.parameters() self.optimizer = torch.optim.Adagrad( parameters, lr=initial_learning_rate, weight_decay=weight_decay, lr_decay=lr_decay, initial_accumulator_value=initial_accumulator_value) print("INITIAL LEARNING RATE: ", initial_learning_rate) print("TOTAL EPOCHS: ", self.args.epochs) print("WEIGHT DECAY: ", weight_decay) print("LR DECAY: ", lr_decay) print("INITIAL ACCUMULATOR VALUE: ", initial_accumulator_value) #self.scheduler = torch.optim.lr_scheduler.MultiStepLR( # self.optimizer, milestones=self.args.learning_rate_decay_epochs, gamma=args.learning_rate_decay_rate) print("\nTRAIN:") #print(self.criterion) print(self.optimizer)
def main(): if args.validation_src == 'FGSM': if args.dataset == 'svhn': out_dist_list = [ 'cifar10', 'imagenet_resize', 'lsun_resize', 'FGSM' ] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize', 'FGSM'] else: if args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] outf_load = os.path.join(args.outf, args.net_type + '_' + args.dataset + 'RealNVP') outf = os.path.join( args.outf, args.net_type + '_' + args.dataset + 'RealNVP_magnitude') if os.path.isdir(outf) == False: os.mkdir(outf) # torch.cuda.manual_seed(0) torch.cuda.set_device(args.cuda_index) if args.dataset == 'cifar100': args.num_classes = 100 else: args.num_classes = 10 with open( 'feature_lists/feature_lists_{}_imagenet_resize_{}_Wlinear.pickle'. format(args.net_type, args.dataset), 'rb') as f: [ sample_class_mean, list_features, list_features_test, list_features_out, A, A_inv, log_abs_det_A_inv ] = pickle.load(f) pre_trained_net = args.net_type + '_' + args.dataset + '.pth' pre_trained_net = os.path.join('pre_trained', pre_trained_net) if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.cuda_index))) else: model = torch.load(pre_trained_net, map_location="cpu") for i, (name, module) in enumerate(model._modules.items()): module = recursion_change_bn(model) for m in model.modules(): if 'Conv' in str(type(m)): setattr(m, 'padding_mode', 'zeros') in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) else: model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.cuda_index))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) print('load model: ' + args.net_type) model.to(device) model.eval() # load dataset print('load in-data: ', args.dataset) num_layers = len(sample_class_mean) for layer in range(num_layers): num_features = A_inv[layer].shape[0] half_features = int(num_features / 2) zeros = np.zeros(half_features) ones = np.ones(half_features) right = np.concatenate((zeros, ones), axis=None) left = np.concatenate((ones, zeros), axis=None) masks = torch.from_numpy( np.array([ right, left, right, left, right, left, right, left, right, left ]).astype(np.float32)).cuda() flow = [] # We reduce the number of neurons in the hidden layers due to GPU memory limitations (11 GB in GTX 2080Ti) - comment out this line for larger GPU memory length_hidden = reture_length_hidden(layer) A_layer = torch.tensor(A[layer]) A_inv_layer = torch.tensor(A_inv[layer]) log_abs_det_A_inv_layer = torch.tensor(log_abs_det_A_inv[layer]) for i in range(args.num_classes): MODEL_FLOW = os.path.join( outf_load, 'model_{}_layer_{}_residual_flow_{}length_hidden'.format( args.dataset, layer, length_hidden), 'flow_{}'.format(i)) flow.append( RealNVP(masks, num_features, length_hidden, A_layer, A_inv_layer, log_abs_det_A_inv_layer)) flow[i].load_state_dict(torch.load(MODEL_FLOW, map_location="cuda:{}".format( args.cuda_index)), strict=False) flow[i].to(device) flow[i].eval() sample_class_mean_layer = sample_class_mean[layer] m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) _, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) score_in = lib_generation.get_resflow_score( test_loader, model, layer, args.num_classes, args.net_type, sample_class_mean_layer, flow, magnitude) for out_dist in out_dist_list: print('load out-data: ', out_dist) if out_dist == 'FGSM': test_loader, out_loader = data_loader.getFGSM( args.batch_size, args.dataset, args.net_type) score_in = lib_generation.get_resflow_score_FGSM( test_loader, model, layer, args.num_classes, args.net_type, sample_class_mean_layer, flow, magnitude) score_out = lib_generation.get_resflow_score_FGSM( out_loader, model, layer, args.num_classes, args.net_type, sample_class_mean_layer, flow, magnitude) else: out_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) score_out = lib_generation.get_resflow_score( out_loader, model, layer, args.num_classes, args.net_type, sample_class_mean_layer, flow, magnitude) pram = { 'out_dist': out_dist, 'Network_type': args.net_type, 'Layer': layer, 'Batch_size': args.batch_size, 'cuda_index': args.cuda_index, 'length_hidden': length_hidden, 'dropout': False, 'weight_decay': 0, 'init_zeros': True, 'num_flows': int(len(flow[0].t)), 'magnitude': magnitude, } with open( os.path.join( outf, 'Residual_flow_%s_%s_layer_%s_%smagnitude.txt' % (args.dataset, out_dist, layer, magnitude)), 'w') as file: file.write('date: %s\n' % (datetime.datetime.now())) file.write(json.dumps(pram)) score_in = np.asarray(score_in, dtype=np.float32) score_out = np.asarray(score_out, dtype=np.float32) score_data, score_labels = lib_generation.merge_and_generate_labels( score_out, score_in) file_name = os.path.join( outf, 'Residual_flow_%s_%s_layer_%s_%smagnitude' % (args.dataset, out_dist, layer, magnitude)) score_data = np.concatenate((score_data, score_labels), axis=1) np.savez(file_name, score_data, pram)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] elif args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] elif args.dataset == 'ham10000': #out_dist_list = ['cifar10', 'imagenet_resize', 'face', 'face_age', 'isic-2017', 'isic-2016'] #out_dist_list = ['cifar10', 'face', 'face_age', 'isic-2017', 'isic-2016'] #out_dist_list = ['cifar10', 'cifar100', 'svhn', 'imagenet_resize', 'lsun_resize', 'face', 'face_age', 'isic-2017', 'isic-2016'] out_dist_list = [ 'ham10000-avg-smoothing', 'ham10000-brightness', 'ham10000-contrast', 'ham10000-dilation', 'ham10000-erosion', 'ham10000-med-smoothing', 'ham10000-rotation', 'ham10000-shift' ] # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) elif args.net_type == 'densenet121': model = DenseNet121(num_classes=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)).state_dict()) in_transform = transforms.Compose([ transforms.Resize(224), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize((0.7630069, 0.5456578, 0.5700767), (0.14093237, 0.15263236, 0.17000099)) ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get Mahalanobis scores', num_output) m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): print('layer_num', i) M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \ True, args.net_type, sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print('Out-distribution: ' + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \ False, args.net_type, sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, out_dist)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth" args.outf = args.outf + args.net_type + "_" + args.dataset + "/" if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == "cifar100": args.num_classes = 100 # load networks if args.net_type == "densenet": if args.dataset == "svhn": model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0), ), ]) elif args.net_type == "resnet": model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print("load model: " + args.net_type) # load dataset print("load target data: ", args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) test_clean_data = torch.load(args.outf + "clean_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_adv_data = torch.load(args.outf + "adv_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_noisy_data = torch.load(args.outf + "noisy_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_label = torch.load(args.outf + "label_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print("get sample mean and covariance") sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print("get LID scores") LID, LID_adv, LID_noisy = lib_generation.get_LID(model, test_clean_data, test_adv_data, test_noisy_data, test_label, num_output) overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90] list_counter = 0 for overlap in overlap_list: Save_LID = np.asarray(LID[list_counter], dtype=np.float32) Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32) Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32) Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy)) LID_data, LID_labels = lib_generation.merge_and_generate_labels( Save_LID_adv, Save_LID_pos) file_name = os.path.join( args.outf, "LID_%s_%s_%s.npy" % (overlap, args.dataset, args.adv_type)) LID_data = np.concatenate((LID_data, LID_labels), axis=1) np.save(file_name, LID_data) list_counter += 1 print("get Mahalanobis scores") m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print("\nNoise: " + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score_adv( model, test_clean_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score_adv( model, test_adv_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy = lib_generation.get_Mahalanobis_score_adv( model, test_noisy_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, "Mahalanobis_%s_%s_%s.npy" % (str(magnitude), args.dataset, args.adv_type), ) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 # load networks if args.net_type == 'densenet': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) test_clean_data, test_adv_data, test_noisy_data, test_label = {}, {}, {}, {} clean_loaders, adv_loaders, noisy_loaders = {}, {}, {} adv_types = ['FGSM', 'BIM', 'DeepFool', 'CWL2', 'PGD100' ] if args.adv_type is None else [args.adv_type] for adv_type in adv_types: test_clean_data[adv_type] = torch.load( args.outf + 'clean_data_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) test_adv_data[adv_type] = torch.load( args.outf + 'adv_data_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) test_noisy_data[adv_type] = torch.load( args.outf + 'noisy_data_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) test_label[adv_type] = torch.load( args.outf + 'label_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) clean_loaders[adv_type] = torch.utils.data.DataLoader( test_clean_data[adv_type], batch_size=args.batch_size) adv_loaders[adv_type] = torch.utils.data.DataLoader( test_adv_data[adv_type], batch_size=args.batch_size) noisy_loaders[adv_type] = torch.utils.data.DataLoader( test_noisy_data[adv_type], batch_size=args.batch_size) # get the feature maps' sizes model.eval() temp_x = torch.rand(1, 3, 32, 32).cuda() feature_maps = model.feature_list(temp_x)[1] feature_map_sizes = [] for out in feature_maps: feature_map_sizes.append(out.size()[1:]) opt_class = torch.optim.Adam criterion = torch.nn.MSELoss(reduction='none') ae_type = args.ae_type filters = [128, 128, 128] arch_text = str(filters).replace(', ', '_').replace('(', '').replace(')', '') latent_size = 64 bn = False if ae_type == 'vae': lamb = 1e-7 elif ae_type == 'wae': lamb = 1e-4 elif ae_type == 'waegan': if args.net_type == 'densenet': lamb = [5e-5, 1e-5, 1e-5, 1e-3] elif args.net_type == 'resnet': lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4] else: lamb = None lr = 1e-3 for run_n in range(args.runs): if ae_type == 'waegan': ae_models = [ build_ae_for_size(ae_type, sizes, filters, latent_size, lam, bn, info=True) for sizes, lam in zip(feature_map_sizes, lamb) ] ae_opts = [] for ae_model in ae_models: ae_opts.append( tuple([ opt_class(ae_model[i].parameters(), lr) for i in range(len(ae_model)) ])) else: ae_models = [ build_ae_for_size(ae_type, sizes, filters, latent_size, lamb, bn, info=True) for sizes in feature_map_sizes ] ae_opts = [ opt_class(ae_model.parameters(), lr) for ae_model in ae_models ] run_name = (f'{args.dataset}_{args.net_type}_deep_{ae_type}_arch_{arch_text}_bn_{bn}' \ f'_latent_{latent_size}_lamb_{lamb}_lr_{lr}_bs_{args.batch_size}_epochs_{args.epochs}_{run_n}') \ .replace('.', '_').replace(',', '_') print(f'Run name: {run_name}') writer = SummaryWriter(run_name) for ae_model in ae_models: if isinstance(ae_model, torch.nn.Module): ae_model.train() else: for m in ae_model: m.train() models_filename = f'{run_name}/models.pth' models_file = Path(models_filename) if not models_file.exists(): current_x = 0 with tqdm(initial=current_x, unit_scale=True, dynamic_ncols=True) as pbar: for epoch in range(args.epochs): for ae_model, ae_opt in zip(ae_models, ae_opts): if isinstance(ae_model, tuple): encoder_o, decoder_o, discriminator_o = ae_opt custom_schedule(encoder_o, epoch) custom_schedule(decoder_o, epoch) custom_schedule(discriminator_o, epoch) elif isinstance(ae_model, ConvWAE): custom_schedule(ae_opt, epoch) elif isinstance(ae_model, ConvVAE): custom_schedule(ae_opt, epoch) for X, _ in train_loader: X = X.cuda() y_pred, feature_list = model.feature_list(X) for i, (feature_map, ae_model, ae_opt) in enumerate( zip(feature_list, ae_models, ae_opts)): feature_map = feature_map.detach() if isinstance(ae_model, tuple): encoder, decoder, discriminator = ae_model encoder_o, decoder_o, discriminator_o = ae_opt z = encoder(feature_map) fm_recon = decoder(z) # train discriminator ones = torch.ones((feature_map.size(0), 1), device='cuda') zeros = torch.zeros((feature_map.size(0), 1), device='cuda') z_sigma = encoder.z_var**0.5 z_fake = torch.empty( (X.size(0), discriminator.z_dim), device='cuda').normal_(std=z_sigma) dis_fake = discriminator(z_fake) dis_real = discriminator(z) fake_loss = F.binary_cross_entropy_with_logits( dis_fake, ones) real_loss = F.binary_cross_entropy_with_logits( dis_real, zeros) d_loss = discriminator.lamb * (fake_loss + real_loss) discriminator_o.zero_grad() d_loss.backward(retain_graph=True) discriminator_o.step() # train autoencoder # rec_losses = criterion(fm_recon, feature_map).sum(dim=(1, 2, 3)) rec_losses = criterion( fm_recon, feature_map).mean(dim=(1, 2, 3)) rec_loss = rec_losses.mean() ae_d_loss = F.binary_cross_entropy_with_logits( dis_real, ones) ae_loss = rec_loss + discriminator.lamb * ae_d_loss encoder_o.zero_grad() decoder_o.zero_grad() ae_loss.backward() encoder_o.step() decoder_o.step() writer.add_scalar(f'AE {i}/AE Loss', ae_loss.item(), global_step=current_x) writer.add_scalar( f'AE {i}/AE Reconstruction loss', rec_loss.item(), global_step=current_x) writer.add_scalar( f'AE {i}/AE discriminator loss', ae_d_loss.item(), global_step=current_x) writer.add_scalar(f'AE {i}/Discriminator loss', d_loss.item(), global_step=current_x) writer.add_scalar( f'AE {i}/Discriminator fake loss', fake_loss.item(), global_step=current_x) writer.add_scalar( f'AE {i}/Discriminator mean real output', torch.sigmoid(dis_real).mean().item(), global_step=current_x) writer.add_scalar( f'AE {i}/Discriminator mean fake output', torch.sigmoid(dis_fake).mean().item(), global_step=current_x) else: if isinstance(ae_model, ConvVAE): fm_recon, mu, logvar, z = ae_model.forward( feature_map) loss, rec_loss, div_loss = ae_model.calculate_loss( feature_map, fm_recon, criterion, mu, logvar) elif isinstance(ae_model, ConvWAE): fm_recon, z = ae_model.forward(feature_map) z_sigma = ae_model.z_var**0.5 z_gen = torch.empty( (feature_map.size(0), ae_model.z_dim)).normal_( std=z_sigma).cuda() loss, rec_loss, dis_loss = ae_model.calculate_loss( feature_map, fm_recon, criterion, z_gen, z) else: fm_recon, z = ae_model.forward(feature_map) loss, rec_loss = ae_model.calculate_loss( feature_map, fm_recon, criterion) ae_opt.zero_grad() loss.backward() ae_opt.step() if isinstance(ae_model, ConvVAE): writer.add_scalar(f'AE {i}/Loss', loss.item(), global_step=current_x) writer.add_scalar( f'AE {i}/Reconstruction loss', rec_loss.mean().item(), global_step=current_x) writer.add_scalar( f'AE {i}/Divergence loss', div_loss.item(), global_step=current_x) elif isinstance(ae_model, ConvWAE): writer.add_scalar(f'AE {i}/Loss', loss.item(), global_step=current_x) writer.add_scalar( f'AE {i}/Reconstruction loss', rec_loss.mean().item(), global_step=current_x) writer.add_scalar( f'AE {i}/Discrepancy loss', dis_loss.item(), global_step=current_x) else: writer.add_scalar(f'AE {i}/Loss', loss.item(), global_step=current_x) current_x += 1 pbar.set_description(f'Epoch {epoch}') pbar.update() # save models state_dict = {} for i, ae_model in enumerate(ae_models): if isinstance(ae_model, torch.nn.Module): state_dict[f'model_{i}'] = ae_model.state_dict() else: state_dict[f'model_{i}'] = tuple([ ae_model[i].state_dict() for i in range(len(ae_model)) ]) torch.save(state_dict, models_filename) else: state_dict = torch.load(models_filename) for i, _ in enumerate(ae_models): if isinstance(ae_models[i], torch.nn.Module): ae_models[i].load_state_dict(state_dict[f'model_{i}']) else: for j in range(len(ae_models[i])): ae_models[i][j].load_state_dict( state_dict[f'model_{i}'][j]) # push entire datasets through the (trained) classifier and trained autoencoder and save the results for ae_model in ae_models: if isinstance(ae_model, torch.nn.Module): ae_model.eval() else: for m in ae_model: m.eval() # order must be preserved - alternatively use tuples and one list loaders = [train_loader] dataset_names = ['train'] datasets = [] for adv_type in adv_types: loaders.append(clean_loaders[adv_type]) dataset_names.append(f'clean_{adv_type}') loaders.append(adv_loaders[adv_type]) dataset_names.append(f'adv_{adv_type}') loaders.append(noisy_loaders[adv_type]) dataset_names.append(f'noisy_{adv_type}') with torch.no_grad(): for name, loader in zip(dataset_names, loaders): filename = f'{run_name}/ae_encoded_{name}.npy' if not Path(filename).exists(): final_features = [] with tqdm(initial=0, unit_scale=True, dynamic_ncols=True) as pbar: for t in loader: if isinstance(t, torch.Tensor): X = t else: X = t[0] X = X.cuda() y_pred, feature_list = model.feature_list(X) features = [] for i, (feature_map, ae_model) in enumerate( zip(feature_list, ae_models)): feature_map = feature_map.detach() if isinstance(ae_model, ConvVAE): fm_recon, mu, logvar, z = ae_model.forward( feature_map) loss, rec_loss, div_loss = ae_model.calculate_loss( feature_map, fm_recon, criterion, mu, logvar) mu_norm = torch.norm(mu, dim=1) logvar_norm = torch.norm(logvar, dim=1) features.append( torch.cat( [ rec_loss.mean( dim=(1, 2, 3)).unsqueeze(1), mu_norm.unsqueeze(1), logvar_norm.unsqueeze(1) ], dim=1).detach().cpu().numpy()) elif isinstance(ae_model, ConvWAE): fm_recon, z = ae_model.forward(feature_map) z_sigma = ae_model.z_var**0.5 z_gen = torch.empty( (feature_map.size(0), ae_model.z_dim)).normal_( std=z_sigma).cuda() loss, rec_loss, dis_loss = ae_model.calculate_loss( feature_map, fm_recon, criterion, z_gen, z) latent_norm = torch.norm(z, dim=1) features.append( torch.cat( [ rec_loss.mean( dim=(1, 2, 3)).unsqueeze(1), latent_norm.unsqueeze(1) ], dim=1).detach().cpu().numpy()) elif isinstance(ae_model, tuple): z = ae_model[0](feature_map) fm_recon = ae_model[1](z) dis_output = ae_model[2](z) latent_norm = torch.norm(z, dim=1) rec_loss = criterion( fm_recon, feature_map).mean(dim=(1, 2, 3)) features.append( torch.cat( [ rec_loss.unsqueeze(1), latent_norm.unsqueeze(1), dis_output ], dim=1).detach().cpu().numpy()) else: fm_recon, z = ae_model.forward(feature_map) loss, rec_loss = ae_model.calculate_loss( feature_map, fm_recon, criterion) latent_norm = torch.norm(z, dim=1) features.append( torch.cat( [ rec_loss.mean( dim=(1, 2, 3)).unsqueeze(1), latent_norm.unsqueeze(1) ], dim=1).detach().cpu().numpy()) final_features.append( np.concatenate(features, axis=1)) pbar.update() autoencoded_set = np.concatenate(final_features, axis=0) datasets.append(autoencoded_set) np.save(filename, autoencoded_set) print(f'Saving {name} encoded dataset to {filename}') else: print(f'Found {filename}') datasets.append(np.load(filename)) # and save the entire latent encoding filename = f'{run_name}/latent_{name}.npy' if not Path(filename).exists(): final_features = [] with tqdm(initial=0, unit_scale=True, dynamic_ncols=True) as pbar: for t in loader: if isinstance(t, torch.Tensor): X = t else: X = t[0] X = X.cuda() y_pred, feature_list = model.feature_list(X) features = [] for i, (feature_map, ae_model) in enumerate( zip(feature_list, ae_models)): feature_map = feature_map.detach() if isinstance(ae_model, ConvVAE): fm_recon, mu, logvar, z = ae_model.forward( feature_map) elif isinstance(ae_model, ConvWAE): fm_recon, z = ae_model.forward(feature_map) elif isinstance(ae_model, tuple): z = ae_model[0](feature_map) else: fm_recon, z = ae_model.forward(feature_map) features.append(z.detach().cpu().numpy()) final_features.append( np.concatenate(features, axis=1)) pbar.update() autoencoded_set = np.concatenate(final_features, axis=0) datasets.append(autoencoded_set) np.save(filename, autoencoded_set) print( f'Saving {name} latent encoded dataset to {filename}') else: print(f'Found {filename}') # test the model performance when features are replaced by the autoencoded features def test_ae_dataset(loader): model.eval() for ae_model in ae_models: if isinstance(ae_model, torch.nn.Module): ae_model.eval() else: for m in ae_model: m.eval() with torch.no_grad(): with tqdm(initial=0, unit_scale=True, dynamic_ncols=True) as pbar: total = 0 keys = ['original', 'all' ] + [i + 1 for i in range(len(ae_models))] correct = {k: 0 for k in keys} masks = {k: [False] * len(ae_models) for k in keys} masks['all'] = [True] * len(ae_models) for i in range(len(ae_models)): masks[i + 1][i] = True for X, y in loader: X = X.cuda() y = y.cuda() total += y.size(0) for k in keys: y_pred = model.ae_replaced_forward( X, ae_models, masks[k]) y_pred_max = y_pred.argmax(dim=1) correct[k] += (y_pred_max == y).sum().item() pbar.update() accs = {k: correct[k] / total for k in keys} return accs train_accs = test_ae_dataset(train_loader) print(f'Trainset accuracy:') for k, v in train_accs.items(): print(f'{k}: {v}') test_accs = test_ae_dataset(test_loader) print(f'Testset accuracy:') for k, v in test_accs.items(): print(f'{k}: {v}')
if args.noise_fraction > 0: print('load noisy labels') args.label_root = args.label_root + args.dataset + '/' + args.noise_type + '/' + str( args.noise_fraction) if os.path.isdir(args.label_root) == False: print('Err: generate noisy labels first') else: args.label_root = args.label_root + '/train_labels.npy' new_label = torch.load(args.label_root) train_loader.dataset.train_labels = new_label print('Model: ', args.net_type) if args.net_type == 'densenet': model = models.DenseNet3(100, int(args.numclass)) elif args.net_type == 'resnet34': model = models.ResNet34(num_c=args.numclass) if args.cuda: model.cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.wd) decreasing_lr = list(map(int, args.decreasing_lr.split(','))) def train(epoch): model.train()
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 if args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize',] #'lsun_resize'] else: out_dist_list = ['svhn', 'imagenet_resize',] #'lsun_resize'] # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict(torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu)) in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([0.5071, 0.4867, 0.4408], [0.2675, 0.2565, 0.2761]),]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict(torch.load(pre_trained_net, map_location = "cuda:" + str(args.gpu))) in_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) # measure the performance M_list = [0, 0.0005, 0.001, 0.0014, 0.002, 0.0024, 0.005, 0.01, 0.05, 0.1, 0.2] T_list = [1, 10, 100, 1000] base_line_list = [] ODIN_best_tnr = [0, 0, 0] ODIN_best_results = [0 , 0, 0] ODIN_best_temperature = [-1, -1, -1] ODIN_best_magnitude = [-1, -1, -1] for T in T_list: for m in M_list: magnitude = m temperature = T lib_generation.get_posterior(model, args.net_type, test_loader, magnitude, temperature, args.outf, True) out_count = 0 print('Temperature: ' + str(temperature) + ' / noise: ' + str(magnitude)) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet(out_dist, args.batch_size, in_transform, args.dataroot) print('Out-distribution: ' + out_dist) lib_generation.get_posterior(model, args.net_type, out_test_loader, magnitude, temperature, args.outf, False) if temperature == 1 and magnitude == 0: test_results = callog.metric(args.outf, ['PoT']) base_line_list.append(test_results) else: val_results = callog.metric(args.outf, ['PoV']) if ODIN_best_tnr[out_count] < val_results['PoV']['TNR']: ODIN_best_tnr[out_count] = val_results['PoV']['TNR'] ODIN_best_results[out_count] = callog.metric(args.outf, ['PoT']) ODIN_best_temperature[out_count] = temperature ODIN_best_magnitude[out_count] = magnitude out_count += 1 # print the results mtypes = ['TNR', 'AUROC', 'DTACC', 'AUIN', 'AUOUT'] print('Baseline method: in_distribution: ' + args.dataset + '==========') count_out = 0 for results in base_line_list: print('out_distribution: '+ out_dist_list[count_out]) for mtype in mtypes: print(' {mtype:6s}'.format(mtype=mtype), end='') print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='') print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='') print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='') print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='') print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='') print('') count_out += 1 print('ODIN method: in_distribution: ' + args.dataset + '==========') count_out = 0 for results in ODIN_best_results: print('out_distribution: '+ out_dist_list[count_out]) for mtype in mtypes: print(' {mtype:6s}'.format(mtype=mtype), end='') print('\n{val:6.2f}'.format(val=100.*results['PoT']['TNR']), end='') print(' {val:6.2f}'.format(val=100.*results['PoT']['AUROC']), end='') print(' {val:6.2f}'.format(val=100.*results['PoT']['DTACC']), end='') print(' {val:6.2f}'.format(val=100.*results['PoT']['AUIN']), end='') print(' {val:6.2f}\n'.format(val=100.*results['PoT']['AUOUT']), end='') print('temperature: ' + str(ODIN_best_temperature[count_out])) print('magnitude: '+ str(ODIN_best_magnitude[count_out])) print('') count_out += 1
if args.dataset == 'cifar100': num_val = 5 total_train_data, total_train_label, _, _ = data_loader.get_raw_data( train_loader, num_classes, 0) total_test_data, total_test_label, val_index, test_index \ = data_loader.get_raw_data(test_loader, num_classes, num_val) total_val_data, total_val_label, total_test_data, total_test_label \ = data_loader.get_validation(total_test_data, total_test_label, val_index, test_index) print('load networks: ' + args.net_type) if args.net_type == 'resnet34': model = models.ResNet34(num_c=num_classes) elif args.net_type == 'densenet': model = models.DenseNet3(100, int(num_classes)) model.load_state_dict( torch.load(file_root + 'model.pth', map_location="cuda:" + str(args.gpu))) model.cuda() print('extact features') utils.extract_features(model, total_train_data, total_train_label, file_root, "train_val") utils.extract_features(model, total_val_data, total_val_label, file_root, "test_val") utils.extract_features(model, total_test_data, total_test_label, file_root, "test_test") test_data_list, test_label_list = [], [] test_val_data_list, test_val_label_list = [], [] train_data_list, train_label_list = [], []
def main(): run_dir = args.run_dir run_name = run_dir.name matches = runs_regex.match(run_name) dataset = matches.group(1) net_type = matches.group(2) ae_type = matches.group(3) args.num_classes = 100 if dataset == 'cifar100' else 10 # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + net_type + '_' + dataset + '.pth' torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) device = f'cuda:{args.gpu}' # load networks if net_type == 'densenet': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + net_type) # load dataset print('load target data: ', dataset) train_loader, test_loader = data_loader.getTargetDataSet( dataset, args.batch_size, in_transform, args.dataroot) # get the feature maps' sizes model.eval() temp_x = torch.rand(1, 3, 32, 32).cuda() feature_maps = model.feature_list(temp_x)[1] feature_map_sizes = [] for out in feature_maps: feature_map_sizes.append(out.size()[1:]) criterion = torch.nn.CrossEntropyLoss() ae_criterion = torch.nn.MSELoss(reduction='none') ae_type = ae_type filters = [128, 128, 128] latent_size = 64 bn = False if ae_type == 'vae': lamb = 1e-7 elif ae_type == 'wae': lamb = 1e-4 elif ae_type == 'waegan': if net_type == 'densenet': lamb = [5e-5, 1e-5, 1e-5, 1e-3] elif net_type == 'resnet': lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4] else: lamb = None if ae_type == 'waegan': ae_models = [ build_ae_for_size(ae_type, sizes, filters, latent_size, lam, bn) for sizes, lam in zip(feature_map_sizes, lamb) ] else: ae_models = [ build_ae_for_size(ae_type, sizes, filters, latent_size, lamb, bn) for sizes in feature_map_sizes ] # read in model and autoencoder models models_filename = f'{run_name}/models.pth' state_dict = torch.load(models_filename) for i, _ in enumerate(ae_models): if isinstance(ae_models[i], torch.nn.Module): ae_models[i].load_state_dict(state_dict[f'model_{i}']) else: for j in range(len(ae_models[i])): ae_models[i][j].load_state_dict(state_dict[f'model_{i}'][j]) for ae_model in ae_models: if isinstance(ae_model, torch.nn.Module): ae_model.eval() else: for m in ae_model: m.eval() # read in final unsupervised model classifier_model = 'IF' run_dir = args.run_dir name_prefix = 'latent_' if args.latent else '' name_prefix += 'rec_error_' if args.rec_error else '' name_prefix += 'both_' if args.both else '' model_filename = str(run_dir / f'{name_prefix}final_cv_{classifier_model}.joblib') assert Path(model_filename).exists() if_classfier = joblib.load(model_filename) pgd_iters = [(i + 1) * 10 for i in range(20)] results_file = run_dir / 'increasing_strength_pgd.pickle' if results_file.exists(): with open(results_file, 'rb') as results_fd: results = pickle.load(results_fd) else: results = {} for iters in pgd_iters: # calculate only if result is not present already key = f'PGD{iters}' if key not in results: # read in adversarial AE encoded samples dataset_names = [f'clean_{key}', f'adv_{key}', f'noisy_{key}'] datasets = {} for name in dataset_names: dataset_path = run_dir / f'ae_encoded_{name}.npy' if dataset_path.exists(): datasets[name] = np.load(str(dataset_path)) else: print(f'{dataset_path} is missing!') test_size = len(datasets[f'clean_{key}']) X_test = np.concatenate([ datasets[f'clean_{key}'], datasets[f'adv_{key}'], datasets[f'noisy_{key}'], ]) y_test = np.concatenate([ np.ones(test_size), np.zeros(test_size), np.ones(test_size), ]) # classify with the unsupervised classifier y_pred = if_classfier.predict(X_test) try: y_scores = if_classfier.decision_function(X_test) except: y_scores = if_classfier.predict_proba(X_test)[0] # calculate scores acc = accuracy_score(y_test, y_pred) auroc = roc_auc_score(y_test, y_scores) results[f'acc_{key}'] = acc results[f'auroc_{key}'] = auroc # save results with open(results_file, 'wb') as results_fd: pickle.dump(results, results_fd) # gather data to numpy arrays xs = np.array(pgd_iters) ys = np.zeros(len(xs)) ys_acc = np.zeros(len(xs)) for i, iters in enumerate(pgd_iters): key = f'PGD{iters}' ys[i] = results[f'auroc_{key}'] ys_acc[i] = results[f'acc_{key}'] # generate plots plot_filename = run_dir / 'increasing_strength_pgd.png' fig, ax1 = plt.subplots(1, 1, figsize=(16, 9)) color = 'tab:red' ax1.set_xlabel('PGD iterations') ax1.set_ylabel('AUROC', color=color) ax1.plot(xs, ys, color=color) ax1.tick_params(axis='y', labelcolor=color) ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis color = 'tab:blue' ax2.set_ylabel('Accuracy', color=color) # we already handled the x-label with ax1 ax2.plot(xs, ys_acc, color=color) ax2.tick_params(axis='y', labelcolor=color) fig.tight_layout() # otherwise the right y-label is slightly clipped print(f'Writing {plot_filename}') fig.savefig(plot_filename, bbox_inches='tight') plot_filename = run_dir / 'auroc_increasing_strength_pgd.png' fig, ax1 = plt.subplots(1, 1, figsize=(16, 9)) ax1.set_xlabel('PGD iterations') ax1.set_ylabel('AUROC') x_ticks = xs[::2] ax1.set_xticks(x_ticks) ax1.set_xticklabels([str(x) for x in x_ticks]) ax1.plot(xs, ys, marker='o', linestyle='solid', markeredgecolor='b') fig.tight_layout() # otherwise the right y-label is slightly clipped print(f'Writing {plot_filename}') fig.savefig(plot_filename, bbox_inches='tight')
def main(): run_dir = args.run_dir run_name = run_dir.name matches = runs_regex.match(run_name) dataset = matches.group(1) net_type = matches.group(2) ae_type = matches.group(3) args.num_classes = 100 if dataset == 'cifar100' else 10 # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + net_type + '_' + dataset + '.pth' torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) device = f'cuda:{args.gpu}' # load networks if net_type == 'densenet': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + net_type) # load dataset print('load target data: ', dataset) train_loader, test_loader = data_loader.getTargetDataSet( dataset, args.batch_size, in_transform, args.dataroot) # get the feature maps' sizes model.eval() temp_x = torch.rand(1, 3, 32, 32).cuda() feature_maps = model.feature_list(temp_x)[1] feature_map_sizes = [] for out in feature_maps: feature_map_sizes.append(out.size()[1:]) criterion = torch.nn.CrossEntropyLoss() ae_criterion = torch.nn.MSELoss(reduction='none') ae_type = ae_type filters = [128, 128, 128] latent_size = 64 bn = False if ae_type == 'vae': lamb = 1e-7 elif ae_type == 'wae': lamb = 1e-4 elif ae_type == 'waegan': if net_type == 'densenet': lamb = [5e-5, 1e-5, 1e-5, 1e-3] elif net_type == 'resnet': lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4] else: lamb = None if ae_type == 'waegan': ae_models = [ build_ae_for_size(ae_type, sizes, filters, latent_size, lam, bn) for sizes, lam in zip(feature_map_sizes, lamb) ] else: ae_models = [ build_ae_for_size(ae_type, sizes, filters, latent_size, lamb, bn) for sizes in feature_map_sizes ] # read in model and autoencoder models models_filename = f'{run_name}/models.pth' state_dict = torch.load(models_filename) for i, _ in enumerate(ae_models): if isinstance(ae_models[i], torch.nn.Module): ae_models[i].load_state_dict(state_dict[f'model_{i}']) else: for j in range(len(ae_models[i])): ae_models[i][j].load_state_dict(state_dict[f'model_{i}'][j]) for ae_model in ae_models: if isinstance(ae_model, torch.nn.Module): ae_model.eval() else: for m in ae_model: m.eval() # read in autoencoded datasets for visualization dataset_names = ['train'] for adv_type in adv_types: dataset_names.append(f'clean_{adv_type}') dataset_names.append(f'adv_{adv_type}') dataset_names.append(f'noisy_{adv_type}') datasets = {} for name in dataset_names: dataset_path = run_dir / f'ae_encoded_{name}.npy' if dataset_path.exists(): datasets[name] = np.load(str(dataset_path)) else: print(f'{dataset_path} is missing!') samples_iterate = 100 test_loader_iter = iter(test_loader) X, y = next(test_loader_iter) X, y = X.cuda(), y.cuda() # plot plot_filename = f'{str(run_dir)}/adv_visualize.png' kde_plot_filename = f'{str(run_dir)}/adv_visualize_kde.png' labels = ['1', '34', '67', '99'] if 'densenet' in str(run_dir) else [ '1', '7', '15', '27', '33' ] fig, ax = plt.subplots(len(ae_models), len(adv_types), figsize=(16 * len(adv_types), 9 * len(labels))) kde_fig, kde_ax = plt.subplots(len(ae_models), len(adv_types), figsize=(16 * len(adv_types), 9 * len(labels))) clean_samples = 250 for l, adv_type in enumerate(adv_types): print(f'generating adv type: {adv_type}') k = l while True: sample_x, sample_y = X[k].unsqueeze(0), y[k].unsqueeze(0) # run the sample through the model y_pred, feature_list = model.feature_list(sample_x) # check if it is classified properly, get another sample if not if torch.argmax(y_pred) == sample_y: inputs = sample_x.repeat(samples_iterate, 1, 1, 1) else: k += 1 print(f'torch.argmax(y_pred): {torch.argmax(y_pred)}; ' f'sample_y: {sample_y}; increasing k to: {k}') continue # generate adversarial examples inputs.requires_grad_() output = model(inputs) loss = criterion(output, y) model.zero_grad() loss.backward() if net_type == 'densenet': min_pixel = -1.98888885975 max_pixel = 2.12560367584 elif net_type == 'resnet': min_pixel = -2.42906570435 max_pixel = 2.75373125076 if adv_type == 'FGSM': # increasing epsilon # adv_noise = 0.05 adv_noise = torch.linspace(0.0, 0.1, steps=samples_iterate, device=device).view(-1, 1, 1, 1) gradient = torch.ge(inputs.grad.data, 0) gradient = (gradient.float() - 0.5) * 2 if net_type == 'densenet': gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \ gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0 / 255.0)) gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \ gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1 / 255.0)) gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \ gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7 / 255.0)) else: gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \ gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023)) gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \ gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994)) gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \ gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010)) # adv_data = torch.add(inputs.data, adv_noise, gradient) perturbations = gradient * adv_noise adv_data = inputs.data + perturbations adv_data = torch.clamp(adv_data, min_pixel, max_pixel).detach() elif adv_type == 'BIM': # adv_noise = 0.01 adv_noise = torch.linspace(0.0, 0.02, steps=samples_iterate, device=device).view(-1, 1, 1, 1) gradient = torch.sign(inputs.grad.data) for _ in range(5): perturbations = gradient * adv_noise inputs = inputs + perturbations inputs = torch.clamp(inputs, min_pixel, max_pixel).detach() inputs.requires_grad_() output = model(inputs) loss = criterion(output, y) model.zero_grad() loss.backward() gradient = torch.sign(inputs.grad.data) if net_type == 'densenet': gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \ gradient.index_select(1, torch.LongTensor([0]).cuda()) / (63.0 / 255.0)) gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \ gradient.index_select(1, torch.LongTensor([1]).cuda()) / (62.1 / 255.0)) gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \ gradient.index_select(1, torch.LongTensor([2]).cuda()) / (66.7 / 255.0)) else: gradient.index_copy_(1, torch.LongTensor([0]).cuda(), \ gradient.index_select(1, torch.LongTensor([0]).cuda()) / (0.2023)) gradient.index_copy_(1, torch.LongTensor([1]).cuda(), \ gradient.index_select(1, torch.LongTensor([1]).cuda()) / (0.1994)) gradient.index_copy_(1, torch.LongTensor([2]).cuda(), \ gradient.index_select(1, torch.LongTensor([2]).cuda()) / (0.2010)) perturbations = gradient * adv_noise adv_data = inputs.data + perturbations adv_data = torch.clamp(adv_data, min_pixel, max_pixel).detach() elif adv_type == 'DeepFool': if net_type == 'resnet': if dataset == 'cifar10': adv_noise_base = 0.18 elif dataset == 'cifar100': adv_noise_base = 0.03 else: adv_noise_base = 0.1 else: if dataset == 'cifar10': adv_noise_base = 0.6 elif dataset == 'cifar100': adv_noise_base = 0.1 else: adv_noise_base = 0.5 adv_data = inputs for i, adv_noise in enumerate( np.linspace(0.0, adv_noise_base * 2, samples_iterate)): sample_x_expanded = sample_x.expand(2, -1, -1, -1) sample_y_expanded = sample_y.expand(2) _, adv_example = adversary.deepfool(model, sample_x_expanded.data.clone(), sample_y_expanded.data.cpu(), \ args.num_classes, step_size=adv_noise, train_mode=False) adv_example = adv_example.cuda() adv_data[i] = adv_example[0] elif adv_type.startswith('PGD'): pgd_iters = int(re.match('PGD(\d+)', adv_type).group(1)) adv_noise_base = 0.05 adv_data = inputs for i, adv_noise in enumerate( np.linspace(0.0, adv_noise_base * 2, samples_iterate)): sample_x_expanded = sample_x.expand(2, -1, -1, -1) sample_y_expanded = sample_y.expand(2) perturbation = pgd_linf(model, sample_x_expanded, sample_y_expanded, adv_noise, 1e-2, 100) adv_data[i] = sample_x_expanded[0] + perturbation[0] else: raise ValueError(f'Unsupported adv_type value: {adv_type}') y_adv, adv_feature_list = model.feature_list(adv_data) if torch.argmax(y_adv[-1]) == sample_y: k += 1 print(f'torch.argmax(y_adv[-1]): {torch.argmax(y_adv[-1])}; ' f'sample_y: {sample_y}; increasing k to: {k}') continue break # get AE features for the created adv examples features = [] for j, (ae_model, feature_map) in enumerate(zip(ae_models, adv_feature_list)): feature_map = feature_map.detach() if isinstance(ae_model, ConvVAE): fm_recon, mu, logvar, z = ae_model.forward(feature_map) loss, rec_loss, div_loss = ae_model.calculate_loss( feature_map, fm_recon, ae_criterion, mu, logvar) mu_norm = torch.norm(mu, dim=1) logvar_norm = torch.norm(logvar, dim=1) features.append( torch.cat([ rec_loss.mean(dim=(1, 2, 3)).unsqueeze(1), mu_norm.unsqueeze(1), logvar_norm.unsqueeze(1) ], dim=1).detach().cpu().numpy()) elif isinstance(ae_model, ConvWAE): fm_recon, z = ae_model.forward(feature_map) z_sigma = ae_model.z_var**0.5 z_gen = torch.empty( (feature_map.size(0), ae_model.z_dim)).normal_(std=z_sigma).cuda() loss, rec_loss, dis_loss = ae_model.calculate_loss( feature_map, fm_recon, ae_criterion, z_gen, z) latent_norm = torch.norm(z, dim=1) features.append( torch.cat([ rec_loss.mean(dim=(1, 2, 3)).unsqueeze(1), latent_norm.unsqueeze(1) ], dim=1).detach().cpu().numpy()) elif isinstance(ae_model, tuple): z = ae_model[0](feature_map) fm_recon = ae_model[1](z) dis_output = ae_model[2](z) latent_norm = torch.norm(z, dim=1) rec_loss = ae_criterion(fm_recon, feature_map).mean(dim=(1, 2, 3)) features.append( torch.cat([ rec_loss.unsqueeze(1), latent_norm.unsqueeze(1), dis_output ], dim=1).detach().cpu().numpy()) else: fm_recon, z = ae_model.forward(feature_map) loss, rec_loss = ae_model.calculate_loss( feature_map, fm_recon, ae_criterion) latent_norm = torch.norm(z, dim=1) features.append( torch.cat([ rec_loss.mean(dim=(1, 2, 3)).unsqueeze(1), latent_norm.unsqueeze(1) ], dim=1).detach().cpu().numpy()) # generate and save figures colors = np.zeros((adv_data.shape[0], 3)) colors[:, 1] = np.linspace(1, 0, adv_data.shape[0]) colors[:, 0] = 1 for j in range(len(features)): # additionally plot multiple points (or KDE) from training/clean data f1_i = j * 2 f2_i = f1_i + 1 clean_xs = datasets['train'][:, f1_i][:clean_samples] clean_ys = datasets['train'][:, f2_i][:clean_samples] ax[j, l].scatter(clean_xs, clean_ys, color='green') sns.kdeplot(clean_xs, clean_ys, cmap='Greens', shade=True, shade_lowest=False, ax=kde_ax[j, l], alpha=1.0) ax[j, l].scatter(features[j][:, 0], features[j][:, 1], color=colors) kde_ax[j, l].scatter(features[j][:, 0], features[j][:, 1], color=colors) ax[j, l].grid(True) kde_ax[j, l].grid(True) ax[j, l].set_title(f'{adv_type} on layer {labels[j]}') kde_ax[j, l].set_title(f'{adv_type} on layer {labels[j]}') ax[j, l].locator_params(nbins=5) kde_ax[j, l].locator_params(nbins=5) print(f'Writing {plot_filename}') fig.savefig(plot_filename, bbox_inches='tight') close(fig) print(f'Writing {kde_plot_filename}') kde_fig.savefig(kde_plot_filename, bbox_inches='tight') close(kde_fig) print(f'FINISHED {str(run_dir)}!')
def main(): if os.path.isdir('feature_lists') == False: os.mkdir('feature_lists') if args.dataset == 'cifar100': args.num_classes = 100 else: args.num_classes = 10 # load networks pre_trained_net = args.net_type + '_' + args.dataset + '.pth' pre_trained_net = os.path.join('pre_trained', pre_trained_net) if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(0))) else: model = torch.load(pre_trained_net, map_location="cpu") for i, (name, module) in enumerate(model._modules.items()): module = recursion_change_bn(model) for m in model.modules(): if 'Conv' in str(type(m)): setattr(m, 'padding_mode', 'zeros') in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(0))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) if args.validation_src == 'FGSM': if args.dataset == 'svhn': out_dist_list = [ 'cifar10', 'imagenet_resize', 'lsun_resize', 'FGSM' ] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize', 'FGSM'] else: if args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] print('load model: ' + args.net_type) model.to(device) model.eval() # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extraction temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 correct, total = 0, 0 num_output = len(feature_list) num_sample_per_class = np.empty(args.num_classes) num_sample_per_class.fill(0) list_features = [] list_features_test = [] list_features_out = [] for i in range(num_output): temp_list = [] list_features_test.append(0) list_features_out.append(0) for j in range(args.num_classes): temp_list.append(0) list_features.append(temp_list) for data, target in train_loader: total += data.size(0) data = data.cuda() data = Variable(data, volatile=True) output, out_features = model.feature_list(data) # get hidden features for i in range(num_output): out_features[i] = out_features[i].view(out_features[i].size(0), out_features[i].size(1), -1) out_features[i] = torch.mean(out_features[i].data, 2) # compute the accuracy pred = output.data.max(1)[1] equal_flag = pred.eq(target.cuda()).cpu() correct += equal_flag.sum() # construct the sample matrix for i in range(data.size(0)): label = target[i] if num_sample_per_class[label] == 0: out_count = 0 for out in out_features: list_features[out_count][label] = out[i].view(1, -1) out_count += 1 else: out_count = 0 for out in out_features: list_features[out_count][label] = torch.cat( (list_features[out_count][label], out[i].view(1, -1)), 0) out_count += 1 num_sample_per_class[label] += 1 sample_class_mean = [] out_count = 0 for num_feature in feature_list: temp_list = torch.Tensor(args.num_classes, int(num_feature)).cuda() for j in range(args.num_classes): temp_list[j] = torch.mean(list_features[out_count][j], 0) sample_class_mean.append(temp_list) out_count += 1 A = [] A_inv = [] log_abs_det_A_inv = [] for k in range(num_output): X = 0 for i in range(args.num_classes): if i == 0: X = list_features[k][i] - sample_class_mean[k][i] else: X = torch.cat( (X, list_features[k][i] - sample_class_mean[k][i]), 0) # find inverse u, s, vh = np.linalg.svd((X.cpu().numpy()) / np.sqrt(X.shape[0]), full_matrices=False) covariance_real = np.cov(X.cpu().numpy().T) valid_indx = s > 1e-5 if (valid_indx.sum() % 2 > 0): valid_indx[valid_indx.sum() - 1] = False covriance_cal = np.matmul( np.matmul(vh[valid_indx, :].transpose(), np.diag(s[valid_indx]**2)), vh[valid_indx, :]) A_temp = np.matmul(vh[valid_indx, :].transpose(), np.diag(s[valid_indx])) A.append(A_temp) covriance_cal2 = np.matmul(A_temp, A_temp.transpose()) s_inv = 1 / s[valid_indx] A_inv_temp = np.matmul(np.diag(s_inv), vh[valid_indx, :]) A_inv.append(A_inv_temp) log_abs_det_A_inv_temp = np.sum(np.log(np.abs(s_inv))) log_abs_det_A_inv.append(log_abs_det_A_inv_temp) print('\n Training Accuracy:({:.2f}%)\n'.format(100.0 * int(correct) / int(total))) num_sample_per_output = np.empty(num_output) num_sample_per_output.fill(0) for data, target in test_loader: data = data.cuda() data = Variable(data, volatile=True) output, out_features = model.feature_list(data) # get hidden features for i in range(num_output): out_features[i] = out_features[i].view(out_features[i].size(0), out_features[i].size(1), -1) out_features[i] = torch.mean(out_features[i].data, 2) if num_sample_per_output[i] == 0: list_features_test[i] = out_features[i] else: list_features_test[i] = torch.cat( (list_features_test[i], out_features[i]), 0) num_sample_per_output[i] += 1 for out_dist in out_dist_list: if out_dist == 'FGSM': test_loader, out_test_loader = data_loader.getFGSM( args.batch_size, args.dataset, args.net_type) num_sample_per_output.fill(0) for data in test_loader: data = data.cuda() data = Variable(data, volatile=True) output, out_features = model.feature_list(data) # get hidden features for i in range(num_output): out_features[i] = out_features[i].view( out_features[i].size(0), out_features[i].size(1), -1) out_features[i] = torch.mean(out_features[i].data, 2) if num_sample_per_output[i] == 0: list_features_test[i] = out_features[i] else: list_features_test[i] = torch.cat( (list_features_test[i], out_features[i]), 0) num_sample_per_output[i] += 1 num_sample_per_output = np.empty(num_output) num_sample_per_output.fill(0) for data in out_test_loader: data = data.cuda() data = Variable(data, requires_grad=True) output, out_features = model.feature_list(data) # get hidden features for i in range(num_output): out_features[i] = out_features[i].view( out_features[i].size(0), out_features[i].size(1), -1) out_features[i] = torch.mean(out_features[i].data, 2) if num_sample_per_output[i] == 0: list_features_out[i] = out_features[i] else: list_features_out[i] = torch.cat( (list_features_out[i], out_features[i]), 0) num_sample_per_output[i] += 1 for i in range(num_output): sample_class_mean[i] = sample_class_mean[i].cpu() list_features_test[i] = list_features_test[i].cpu() list_features_out[i] = list_features_out[i].cpu() for j in range(args.num_classes): list_features[i][j] = list_features[i][j].cpu() else: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) num_sample_per_output.fill(0) for data, target in out_test_loader: data, target = data.cuda(), target.cuda() data, target = Variable(data, requires_grad=True), Variable(target) output, out_features = model.feature_list(data) # get hidden features for i in range(num_output): out_features[i] = out_features[i].view( out_features[i].size(0), out_features[i].size(1), -1) out_features[i] = torch.mean(out_features[i].data, 2) if num_sample_per_output[i] == 0: list_features_out[i] = out_features[i] else: list_features_out[i] = torch.cat( (list_features_out[i], out_features[i]), 0) num_sample_per_output[i] += 1 for i in range(num_output): sample_class_mean[i] = sample_class_mean[i].cpu() list_features_test[i] = list_features_test[i].cpu() list_features_out[i] = list_features_out[i].cpu() for j in range(args.num_classes): list_features[i][j] = list_features[i][j].cpu() file_name = os.path.join( 'feature_lists', 'feature_lists_{}_{}_{}_Wlinear.pickle'.format( args.net_type, out_dist, args.dataset)) with open(file_name, 'wb') as f: pickle.dump([ sample_class_mean, list_features, list_features_test, list_features_out, A, A_inv, log_abs_det_A_inv ], f)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 # load networks if args.net_type == 'densenet': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) test_clean_data, test_adv_data, test_noisy_data, test_label = {}, {}, {}, {} clean_loaders, adv_loaders, noisy_loaders = {}, {}, {} adv_types = ['FGSM', 'BIM', 'DeepFool', 'CWL2'] for adv_type in adv_types: test_clean_data[adv_type] = torch.load( args.outf + 'clean_data_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) test_adv_data[adv_type] = torch.load( args.outf + 'adv_data_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) test_noisy_data[adv_type] = torch.load( args.outf + 'noisy_data_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) test_label[adv_type] = torch.load( args.outf + 'label_%s_%s_%s.pth' % (args.net_type, args.dataset, adv_type)) clean_loaders[adv_type] = torch.utils.data.DataLoader( test_clean_data[adv_type], batch_size=args.batch_size) adv_loaders[adv_type] = torch.utils.data.DataLoader( test_adv_data[adv_type], batch_size=args.batch_size) noisy_loaders[adv_type] = torch.utils.data.DataLoader( test_noisy_data[adv_type], batch_size=args.batch_size) # get the feature maps' sizes model.eval() temp_x = torch.rand(1, 3, 32, 32).cuda() feature_maps = model.feature_list(temp_x)[1] feature_map_sizes = [] for out in feature_maps: feature_map_sizes.append(out.size()[1:]) opt_class = torch.optim.Adam criterion = torch.nn.MSELoss(reduction='none') ae_type = args.ae_type filters = [128, 128, 128] arch_text = str(filters).replace(', ', '_').replace('(', '').replace(')', '') latent_size = 64 bn = False if ae_type == 'vae': lamb = 1e-7 elif ae_type == 'wae': lamb = 1e-4 elif ae_type == 'waegan': if args.net_type == 'densenet': lamb = [5e-5, 1e-5, 1e-5, 1e-3] elif args.net_type == 'resnet': lamb = [5e-5, 5e-6, 5e-6, 2e-5, 5e-4] else: lamb = None lr = 1e-3 for run_n in range(args.runs): if ae_type == 'waegan': ae_models = [ build_ae_for_size(ae_type, sizes, filters, latent_size, lam, bn, info=True) for sizes, lam in zip(feature_map_sizes, lamb) ] else: ae_models = [ build_ae_for_size(ae_type, sizes, filters, latent_size, lamb, bn, info=True) for sizes in feature_map_sizes ] run_name = (f'{args.dataset}_{args.net_type}_deep_{ae_type}_arch_{arch_text}_bn_{bn}' \ f'_latent_{latent_size}_lamb_{lamb}_lr_{lr}_bs_{args.batch_size}_epochs_{args.epochs}_{run_n}') \ .replace('.', '_').replace(',', '_') print(f'Run name: {run_name}') writer = SummaryWriter(run_name) models_filename = f'{run_name}/models.pth' models_file = Path(models_filename) assert models_file.exists(), 'No saved model available' state_dict = torch.load(models_filename) for i, _ in enumerate(ae_models): if isinstance(ae_models[i], torch.nn.Module): ae_models[i].load_state_dict(state_dict[f'model_{i}']) else: for j in range(len(ae_models[i])): ae_models[i][j].load_state_dict( state_dict[f'model_{i}'][j]) for ae_model in ae_models: if isinstance(ae_model, torch.nn.Module): ae_model.eval() else: for m in ae_model: m.eval() # build reconstruction AE models if ae_type == 'waegan': rec_ae_models = [ build_ae_for_size(ae_type, sizes, filters, latent_size, lam, bn, info=True) for sizes, lam in zip(feature_map_sizes, lamb) ] rec_ae_opts = [] for rec_ae_model in ae_models: rec_ae_opts.append( tuple([ opt_class(rec_ae_model[i].parameters(), lr) for i in range(len(rec_ae_model)) ])) else: rec_ae_models = [ build_ae_for_size(ae_type, sizes, filters, latent_size, lamb, bn, info=True) for sizes in feature_map_sizes ] rec_ae_opts = [ opt_class(rec_ae_model.parameters(), lr) for rec_ae_model in rec_ae_models ] # train reconstruction AE model rec_models_filename = f'{run_name}/rec_models.pth' rec_models_file = Path(rec_models_filename) if not rec_models_file.exists(): current_x = 0 with tqdm(initial=current_x, unit_scale=True, dynamic_ncols=True) as pbar: for epoch in range(args.epochs): for ae_model, rec_ae_model, rec_ae_opt in zip( ae_models, rec_ae_models, rec_ae_opts): if isinstance(ae_model, tuple): encoder_o, decoder_o, discriminator_o = rec_ae_opt custom_schedule(encoder_o, epoch) custom_schedule(decoder_o, epoch) custom_schedule(discriminator_o, epoch) elif isinstance(ae_model, ConvWAE): custom_schedule(rec_ae_opt, epoch) elif isinstance(ae_model, ConvVAE): custom_schedule(rec_ae_opt, epoch) for X, _ in train_loader: X = X.cuda() y_pred, feature_list = model.feature_list(X) for i, (feature_map, ae_model, rec_ae_model, rec_ae_opt) in enumerate( zip(feature_list, ae_models, rec_ae_models, rec_ae_opts)): feature_map = feature_map.detach() # train on x i r(x), so: if isinstance(ae_model, tuple): raise NotImplementedError() else: if isinstance(ae_model, ConvVAE): raise NotImplementedError() elif isinstance(ae_model, ConvWAE): fm_recon, _ = ae_model(feature_map) recon_recon, z = rec_ae_model.forward( fm_recon) z_sigma = rec_ae_model.z_var**0.5 z_gen = torch.empty( (fm_recon.size(0), rec_ae_model.z_dim)).normal_( std=z_sigma).cuda() loss, rec_loss, dis_loss = rec_ae_model.calculate_loss( fm_recon, recon_recon, criterion, z_gen, z) else: fm_recon, _ = ae_model(feature_map) recon_recon, z = rec_ae_model.forward( fm_recon) loss, rec_loss = rec_ae_model.calculate_loss( fm_recon, recon_recon, criterion) rec_ae_opt.zero_grad() loss.backward() rec_ae_opt.step() if isinstance(ae_model, ConvWAE): writer.add_scalar(f'Rec AE {i}/Loss', loss.item(), global_step=current_x) writer.add_scalar( f'Rec AE {i}/Reconstruction loss', rec_loss.mean().item(), global_step=current_x) writer.add_scalar( f'Rec AE {i}/Discrepancy loss', dis_loss.item(), global_step=current_x) else: writer.add_scalar(f'Rec AE {i}/Loss', loss.item(), global_step=current_x) current_x += 1 pbar.set_description(f'Epoch {epoch}') pbar.update() # save rec AE models state_dict = {} for i, rec_ae_model in enumerate(rec_ae_models): state_dict[f'model_{i}'] = rec_ae_model.state_dict() torch.save(state_dict, rec_models_filename) else: state_dict = torch.load(rec_models_filename) for i, _ in enumerate(rec_ae_models): ae_models[i].load_state_dict(state_dict[f'model_{i}']) # order must be preserved - alternatively use tuples and one list loaders = [train_loader] dataset_names = ['train'] datasets = [] for adv_type in adv_types: loaders.append(clean_loaders[adv_type]) dataset_names.append(f'clean_{adv_type}') loaders.append(adv_loaders[adv_type]) dataset_names.append(f'adv_{adv_type}') loaders.append(noisy_loaders[adv_type]) dataset_names.append(f'noisy_{adv_type}') with torch.no_grad(): for name, loader in zip(dataset_names, loaders): filename = f'{run_name}/rec_ae_encoded_{name}.npy' if not Path(filename).exists(): final_features = [] with tqdm(initial=0, unit_scale=True, dynamic_ncols=True) as pbar: for t in loader: if isinstance(t, torch.Tensor): X = t else: X = t[0] X = X.cuda() y_pred, feature_list = model.feature_list(X) features = [] for i, (feature_map, ae_model, rec_ae_model) in enumerate( zip(feature_list, ae_models, rec_ae_models)): feature_map = feature_map.detach() if isinstance(ae_model, ConvVAE): raise NotImplementedError() elif isinstance(ae_model, ConvWAE): fm_recon, _ = ae_model(feature_map) recon_recon, z = rec_ae_model.forward( fm_recon) z_sigma = rec_ae_model.z_var**0.5 z_gen = torch.empty( (fm_recon.size(0), rec_ae_model.z_dim)).normal_( std=z_sigma).cuda() loss, rec_loss, dis_loss = rec_ae_model.calculate_loss( fm_recon, recon_recon, criterion, z_gen, z) latent_norm = torch.norm(z, dim=1) features.append( torch.cat( [ rec_loss.mean( dim=(1, 2, 3)).unsqueeze(1), latent_norm.unsqueeze(1) ], dim=1).detach().cpu().numpy()) elif isinstance(ae_model, tuple): raise NotImplementedError() else: fm_recon, _ = ae_model(feature_map) recon_recon, z = rec_ae_model.forward( fm_recon) loss, rec_loss = rec_ae_model.calculate_loss( fm_recon, recon_recon, criterion) latent_norm = torch.norm(z, dim=1) features.append( torch.cat( [ rec_loss.mean( dim=(1, 2, 3)).unsqueeze(1), latent_norm.unsqueeze(1) ], dim=1).detach().cpu().numpy()) final_features.append( np.concatenate(features, axis=1)) pbar.update() autoencoded_set = np.concatenate(final_features, axis=0) datasets.append(autoencoded_set) np.save(filename, autoencoded_set) print(f'Saving {name} encoded dataset to {filename}') else: print(f'Found {filename}') datasets.append(np.load(filename)) # and save the entire latent encoding filename = f'{run_name}/rec_ae_latent_{name}.npy' if not Path(filename).exists(): final_features = [] with tqdm(initial=0, unit_scale=True, dynamic_ncols=True) as pbar: for t in loader: if isinstance(t, torch.Tensor): X = t else: X = t[0] X = X.cuda() y_pred, feature_list = model.feature_list(X) features = [] for i, (feature_map, ae_model, rec_ae_model) in enumerate( zip(feature_list, ae_models, rec_ae_models)): feature_map = feature_map.detach() if isinstance(ae_model, ConvVAE): raise NotImplementedError() elif isinstance(ae_model, ConvWAE): fm_recon, _ = ae_model(feature_map) recon_recon, z = rec_ae_model.forward( fm_recon) elif isinstance(ae_model, tuple): raise NotImplementedError() else: fm_recon, _ = ae_model(feature_map) recon_recon, z = rec_ae_model.forward( fm_recon) features.append(z.detach().cpu().numpy()) final_features.append( np.concatenate(features, axis=1)) pbar.update() autoencoded_set = np.concatenate(final_features, axis=0) datasets.append(autoencoded_set) np.save(filename, autoencoded_set) print( f'Saving {name} latent encoded dataset to {filename}') else: print(f'Found {filename}')