def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 if args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 ################################ edits print("Calculate SVD before getting sample mean and variance") # svd_result= lib_generation.get_pca(model, args.num_classes, feature_list, train_loader) # lib_generation.get_pca_incremental(model, args.num_classes, feature_list, train_loader,args) svd_result = None print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader, svd_result, args) ################################ edits_end_sample_generator print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \ True, args.net_type, sample_mean, precision, i, magnitude,svd_result,args) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print('Out-distribution: ' + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \ False, args.net_type, sample_mean, precision, i, magnitude,svd_result,args) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, out_dist)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth" args.outf = args.outf + args.net_type + "_" + args.dataset + "/" if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == "cifar100": args.num_classes = 100 if args.dataset == "svhn": out_dist_list = ["cifar10", "imagenet_resize", "lsun_resize"] else: out_dist_list = ["svhn", "imagenet_resize", "lsun_resize"] # load networks if args.net_type == "densenet": if args.dataset == "svhn": model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0), ), ]) elif args.net_type == "resnet": model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print("load model: " + args.net_type) # load dataset print("load target data: ", args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print("get sample mean and covariance") sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print("get Mahalanobis scores") m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print("Noise: " + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score( model, test_loader, args.num_classes, args.outf, True, args.net_type, sample_mean, precision, i, magnitude, ) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print("Out-distribution: " + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score( model, out_test_loader, args.num_classes, args.outf, False, args.net_type, sample_mean, precision, i, magnitude, ) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) ( Mahalanobis_data, Mahalanobis_labels, ) = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, "Mahalanobis_%s_%s_%s.npy" % (str(magnitude), args.dataset, out_dist), ) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) for i, (name, module) in enumerate(model._modules.items()): module = recursion_change_bn(model) for m in model.modules(): if 'Conv' in str(type(m)): setattr(m, 'padding_mode', 'zeros') in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('\nNoise: ' + str(magnitude)) for i in range(num_output): M_in \ = lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out \ = lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy \ = lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, 'FGSM')) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def extract_features(pre_trained_net, in_distribution, in_dist_name, out_dist_list, out_of_distribution, in_transform, gpu, batch_size, num_classes): # set the path to pre-trained model and output outf = "/output/" outf = outf + "model" + '_' + in_dist_name + '/' if os.path.isdir(outf) == False: os.mkdir(outf) torch.cuda.manual_seed(0) torch.cuda.set_device(gpu) # load networks model = torch.load(pre_trained_net, map_location="cuda:" + str(gpu)) model.cuda() print('loaded model') # load target dataset validation_split = .2 shuffle_dataset = True random_seed = 42 # Creating data indices for training and validation splits: dataset_size = len(in_distribution) indices = list(range(dataset_size)) split = int(np.floor(validation_split * dataset_size)) if shuffle_dataset: np.random.seed(random_seed) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] # Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) if in_transform is not None: train_loader = torch.utils.data.DataLoader(in_distribution, batch_size=batch_size, shuffle=True, sampler=train_sampler, **kwargs) test_loader = torch.utils.data.DataLoader(in_distribution, batch_size=batch_size, shuffle=True, sampler=valid_sampler, **kwargs) else: train_loader = torch.utils.data.DataLoader(in_distribution, batch_size=batch_size, shuffle=True, sampler=train_sampler, **kwargs) test_loader = torch.utils.data.DataLoader(in_distribution, batch_size=batch_size, shuffle=True, sampler=valid_sampler, **kwargs) print('loaded target data: ', in_dist_name) # set information about feature extaction model.eval() temp_x = torch.rand(*(list(next(iter(train_loader))[0].size()))).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, num_classes, feature_list, train_loader, model_name="model") print('Generate dataloaders...') out_test_loaders = [] for out_dist in out_of_distribution: out_test_loaders.append( torch.utils.data.DataLoader(out_dist, batch_size=batch_size, shuffle=True, **kwargs)) print('get Mahalanobis scores', num_output) m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): print('layer_num', i) M_in = lib_generation.get_Mahalanobis_score(model, test_loader, num_classes, outf, \ True, "model", sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_test_loader, out_dist in zip(out_test_loaders, out_dist_list): print('Out-distribution: ' + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, num_classes, outf, \ False, "model", sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), in_dist_name, out_dist)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'malaria': args.num_classes = 2 if args.dataset == 'cifar100': args.num_classes = 100 # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) elif args.net_type == 'resnet18': model = models.get_model(args.net_type) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) model.cuda() print('load model: ' + args.net_type) # load dataset # selected_list = torch.LongTensor(selected_list) # test_clean_data = torch.index_select(test_clean_data, 0, selected_list) # test_adv_data = torch.index_select(test_adv_data, 0, selected_list) # test_noisy_data = torch.index_select(test_noisy_data, 0, selected_list) # test_label = torch.index_select(test_label, 0, selected_list) save_path = '%s/%s_%s/' % (args.outf, args.net_type, args.dataset) os.makedirs(save_path, exist_ok=True) # torch.save(test_clean_data, '%s/clean_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_adv_data, '%s/adv_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_noisy_data, '%s/noisy_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_label, '%s/label_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) print('load target data: ', args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, None, args.dataroot) test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get LID scores') LID, LID_adv, LID_noisy \ = lib_generation.get_LID(model, test_clean_data, test_adv_data, test_noisy_data, test_label, num_output) overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90] list_counter = 0 for overlap in overlap_list: Save_LID = np.asarray(LID[list_counter], dtype=np.float32) Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32) Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32) Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy)) LID_data, LID_labels = lib_generation.merge_and_generate_labels( Save_LID_adv, Save_LID_pos) file_name = os.path.join( args.outf, 'LID_%s_%s_%s.npy' % (overlap, args.dataset, args.adv_type)) LID_data = np.concatenate((LID_data, LID_labels), axis=1) np.save(file_name, LID_data) list_counter += 1 print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('\nNoise: ' + str(magnitude)) for i in range(num_output): M_in \ = lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out \ = lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy \ = lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, args.adv_type)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] elif args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] elif args.dataset == 'ham10000': #out_dist_list = ['cifar10', 'imagenet_resize', 'face', 'face_age', 'isic-2017', 'isic-2016'] #out_dist_list = ['cifar10', 'face', 'face_age', 'isic-2017', 'isic-2016'] #out_dist_list = ['cifar10', 'cifar100', 'svhn', 'imagenet_resize', 'lsun_resize', 'face', 'face_age', 'isic-2017', 'isic-2016'] out_dist_list = [ 'ham10000-avg-smoothing', 'ham10000-brightness', 'ham10000-contrast', 'ham10000-dilation', 'ham10000-erosion', 'ham10000-med-smoothing', 'ham10000-rotation', 'ham10000-shift' ] # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) elif args.net_type == 'densenet121': model = DenseNet121(num_classes=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)).state_dict()) in_transform = transforms.Compose([ transforms.Resize(224), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize((0.7630069, 0.5456578, 0.5700767), (0.14093237, 0.15263236, 0.17000099)) ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get Mahalanobis scores', num_output) m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): print('layer_num', i) M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \ True, args.net_type, sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print('Out-distribution: ' + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \ False, args.net_type, sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, out_dist)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth" args.outf = args.outf + args.net_type + "_" + args.dataset + "/" if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == "cifar100": args.num_classes = 100 # load networks if args.net_type == "densenet": if args.dataset == "svhn": model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0), ), ]) elif args.net_type == "resnet": model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print("load model: " + args.net_type) # load dataset print("load target data: ", args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) test_clean_data = torch.load(args.outf + "clean_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_adv_data = torch.load(args.outf + "adv_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_noisy_data = torch.load(args.outf + "noisy_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_label = torch.load(args.outf + "label_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print("get sample mean and covariance") sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print("get LID scores") LID, LID_adv, LID_noisy = lib_generation.get_LID(model, test_clean_data, test_adv_data, test_noisy_data, test_label, num_output) overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90] list_counter = 0 for overlap in overlap_list: Save_LID = np.asarray(LID[list_counter], dtype=np.float32) Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32) Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32) Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy)) LID_data, LID_labels = lib_generation.merge_and_generate_labels( Save_LID_adv, Save_LID_pos) file_name = os.path.join( args.outf, "LID_%s_%s_%s.npy" % (overlap, args.dataset, args.adv_type)) LID_data = np.concatenate((LID_data, LID_labels), axis=1) np.save(file_name, LID_data) list_counter += 1 print("get Mahalanobis scores") m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print("\nNoise: " + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score_adv( model, test_clean_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score_adv( model, test_adv_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy = lib_generation.get_Mahalanobis_score_adv( model, test_noisy_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, "Mahalanobis_%s_%s_%s.npy" % (str(magnitude), args.dataset, args.adv_type), ) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def _generate_Mahalanobis(model, loaders, device, num_classes, model_type='eb0'): model.eval() train_ind_loader, val_ind_loader, test_ind_loader, val_ood_loader, test_ood_loader = loaders temp_x = torch.rand(2, 3, 224, 224).to(device) temp_x = Variable(temp_x) temp_x = temp_x.to(device) if model_type == 'eb0': idxs = [0, 2, 4, 7, 10, 14, 15] x, features = model.extract_features(temp_x, mode='all') features = [features[idx] for idx in idxs] + [x] num_output = len(features) feature_list = np.empty(num_output) count = 0 for out in features: feature_list[count] = out.size(1) count += 1 sample_mean, precision = lib_generation.sample_estimator(model, num_classes, feature_list, train_ind_loader, device=device) m_list = [-0.01, -0.0005, 0.0, 0.0005, 0.01] for magnitude in tqdm(m_list): for i in range(num_output): M_val = lib_generation.get_Mahalanobis_score(model, val_ind_loader, num_classes, sample_mean, precision, i, magnitude, device=device) M_val = np.asarray(M_val, dtype=np.float32) if i == 0: Mahalanobis_val_ind = M_val.reshape((M_val.shape[0], -1)) else: Mahalanobis_val_ind = np.concatenate( (Mahalanobis_val_ind, M_val.reshape((M_val.shape[0], -1))), axis=1) for i in range(num_output): M_val_ood = lib_generation.get_Mahalanobis_score(model, val_ood_loader, num_classes, sample_mean, precision, i, magnitude, device=device) M_val_ood = np.asarray(M_val_ood, dtype=np.float32) if i == 0: Mahalanobis_val_ood = M_val_ood.reshape( (M_val_ood.shape[0], -1)) else: Mahalanobis_val_ood = np.concatenate( (Mahalanobis_val_ood, M_val_ood.reshape((M_val_ood.shape[0], -1))), axis=1) Mahalanobis_val_ind = np.asarray(Mahalanobis_val_ind, dtype=np.float32) Mahalanobis_val_ood = np.asarray(Mahalanobis_val_ood, dtype=np.float32) regressor, _, _ = _score_mahalanobis(Mahalanobis_val_ind, Mahalanobis_val_ood) val_ind = regressor.predict_proba(Mahalanobis_val_ind)[:, 1] val_ood = regressor.predict_proba(Mahalanobis_val_ood)[:, 1] for i in range(num_output): M_test = lib_generation.get_Mahalanobis_score(model, test_ind_loader, num_classes, sample_mean, precision, i, magnitude, device=device) M_test = np.asarray(M_test, dtype=np.float32) if i == 0: Mahalanobis_test = M_test.reshape((M_test.shape[0], -1)) else: Mahalanobis_test = np.concatenate( (Mahalanobis_test, M_test.reshape((M_test.shape[0], -1))), axis=1) for i in range(num_output): M_ood = lib_generation.get_Mahalanobis_score(model, test_ood_loader, num_classes, sample_mean, precision, i, magnitude, device=device) M_ood = np.asarray(M_ood, dtype=np.float32) if i == 0: Mahalanobis_ood = M_ood.reshape((M_ood.shape[0], -1)) else: Mahalanobis_ood = np.concatenate( (Mahalanobis_ood, M_ood.reshape((M_ood.shape[0], -1))), axis=1) Mahalanobis_test = np.asarray(Mahalanobis_test, dtype=np.float32) Mahalanobis_ood = np.asarray(Mahalanobis_ood, dtype=np.float32) ind = regressor.predict_proba(Mahalanobis_test)[:, 1] ood = regressor.predict_proba(Mahalanobis_ood)[:, 1] print(f'########## epsilon: {magnitude} ##########') _ood_detection_performance('Mahalanobis', val_ind, val_ood, ind, ood)
def main(): # set the path to pre-trained model and output args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) out_dist_list = [ 'skin_cli', 'skin_derm', 'corrupted', 'corrupted_70', 'imgnet', 'nct', 'final_test' ] # load networks if args.net_type == 'densenet_121': model = densenet_121.Net(models.densenet121(pretrained=False), 8) ckpt = torch.load("../checkpoints/densenet-121/checkpoint.pth") model.load_state_dict(ckpt['model_state_dict']) model.eval() model.cuda() elif args.net_type == 'mobilenet': model = mobilenet.Net(models.mobilenet_v2(pretrained=False), 8) ckpt = torch.load("../checkpoints/mobilenet/checkpoint.pth") model.load_state_dict(ckpt['model_state_dict']) model.eval() model.cuda() print("Done!") elif args.net_type == 'resnet_50': model = resnet_50.Net(models.resnet50(pretrained=False), 8) ckpt = torch.load("../checkpoints/resnet-50/checkpoint.pth") model.load_state_dict(ckpt['model_state_dict']) model.eval() model.cuda() print("Done!") elif args.net_type == 'vgg_16': model = vgg_16.Net(models.vgg16_bn(pretrained=False), 8) ckpt = torch.load("../checkpoints/vgg-16/checkpoint.pth") model.load_state_dict(ckpt['model_state_dict']) model.eval() model.cuda() print("Done!") else: raise Exception(f"There is no net_type={args.net_type} available.") in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 224, 224).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \ True, args.net_type, sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print('Out-distribution: ' + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \ False, args.net_type, sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, out_dist)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def _generate_Mahalanobis(model, loaders, device, num_classes, model_type='eb0'): model.eval() train_ind_loader, val_ind_loader, test_ind_loader, val_ood_loader, test_ood_loader_1, test_ood_loader_2, test_ood_loader_3 = loaders temp_x = torch.rand(2, 3, 224, 224).to(device) temp_x = Variable(temp_x) temp_x = temp_x.to(device) if model_type == 'eb0': idxs = [0, 2, 4, 7, 10, 14, 15] x, features = model.extract_features(temp_x, mode='all') else: # TODO: In case you wish to evaluate other models, you need to define a proper way to get middle level features pass features = [features[idx] for idx in idxs] + [x] num_output = len(features) feature_list = np.empty(num_output) count = 0 for out in features: feature_list[count] = out.size(1) count += 1 sample_mean, precision = lib_generation.sample_estimator(model, num_classes, feature_list, train_ind_loader, device=device) best_auc = 0 m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] best_magnitudes, best_fprs, regressors, thresholds = [], [], [], [] for magnitude in m_list: for i in range(num_output): M_val = lib_generation.get_Mahalanobis_score(model, val_ind_loader, num_classes, sample_mean, precision, i, magnitude, device=device) M_val = np.asarray(M_val, dtype=np.float32) if i == 0: Mahalanobis_val_ind = M_val.reshape((M_val.shape[0], -1)) else: Mahalanobis_val_ind = np.concatenate( (Mahalanobis_val_ind, M_val.reshape((M_val.shape[0], -1))), axis=1) for i in range(num_output): M_val_ood = lib_generation.get_Mahalanobis_score(model, val_ood_loader, num_classes, sample_mean, precision, i, magnitude, device=device) M_val_ood = np.asarray(M_val_ood, dtype=np.float32) if i == 0: Mahalanobis_val_ood = M_val_ood.reshape( (M_val_ood.shape[0], -1)) else: Mahalanobis_val_ood = np.concatenate( (Mahalanobis_val_ood, M_val_ood.reshape((M_val_ood.shape[0], -1))), axis=1) Mahalanobis_val_ind = np.asarray(Mahalanobis_val_ind, dtype=np.float32) Mahalanobis_val_ood = np.asarray(Mahalanobis_val_ood, dtype=np.float32) regressor, auc, threshold = _score_mahalanobis(Mahalanobis_val_ind, Mahalanobis_val_ood) with open( f'lr_pickles/logistic_regressor_{ind_dataset}_{val_dataset}_{magnitude}.pickle', 'wb') as lrp: pickle.dump(regressor, lrp, protocol=pickle.HIGHEST_PROTOCOL) if auc > best_auc: best_auc = auc best_magnitudes = [magnitude] regressors = [regressor] thresholds = [threshold] best_val_ind = regressor.predict_proba(Mahalanobis_val_ind)[:, 1] best_val_ood = regressor.predict_proba(Mahalanobis_val_ood)[:, 1] cnt = 1 elif auc == best_auc: best_magnitudes.append(magnitude) regressors.append(regressor) thresholds.append(threshold) best_val_ind += regressor.predict_proba(Mahalanobis_val_ind)[:, 1] best_val_ood += regressor.predict_proba(Mahalanobis_val_ood)[:, 1] cnt += 1 best_val_ind /= cnt best_val_ood /= cnt print('###############################################') print() print(f'Selected magnitudes: {best_magnitudes}') print(f'Selected thresholds: {thresholds}') print() idx = 0 for (best_magnitude, regressor, threshold) in zip(best_magnitudes, regressors, thresholds): for i in range(num_output): M_test = lib_generation.get_Mahalanobis_score(model, test_ind_loader, num_classes, sample_mean, precision, i, best_magnitude, device=device) M_test = np.asarray(M_test, dtype=np.float32) if i == 0: Mahalanobis_test = M_test.reshape((M_test.shape[0], -1)) else: Mahalanobis_test = np.concatenate( (Mahalanobis_test, M_test.reshape((M_test.shape[0], -1))), axis=1) for i in range(num_output): M_ood_1 = lib_generation.get_Mahalanobis_score(model, test_ood_loader_1, num_classes, sample_mean, precision, i, best_magnitude, device=device) M_ood_1 = np.asarray(M_ood_1, dtype=np.float32) if i == 0: Mahalanobis_ood_1 = M_ood_1.reshape((M_ood_1.shape[0], -1)) else: Mahalanobis_ood_1 = np.concatenate( (Mahalanobis_ood_1, M_ood_1.reshape( (M_ood_1.shape[0], -1))), axis=1) for i in range(num_output): M_ood_2 = lib_generation.get_Mahalanobis_score(model, test_ood_loader_2, num_classes, sample_mean, precision, i, best_magnitude, device=device) M_ood_2 = np.asarray(M_ood_2, dtype=np.float32) if i == 0: Mahalanobis_ood_2 = M_ood_2.reshape((M_ood_2.shape[0], -2)) else: Mahalanobis_ood_2 = np.concatenate( (Mahalanobis_ood_2, M_ood_2.reshape( (M_ood_2.shape[0], -2))), axis=1) for i in range(num_output): M_ood_3 = lib_generation.get_Mahalanobis_score(model, test_ood_loader_3, num_classes, sample_mean, precision, i, best_magnitude, device=device) M_ood_3 = np.asarray(M_ood_3, dtype=np.float32) if i == 0: Mahalanobis_ood_3 = M_ood_3.reshape((M_ood_3.shape[0], -3)) else: Mahalanobis_ood_3 = np.concatenate( (Mahalanobis_ood_3, M_ood_3.reshape( (M_ood_3.shape[0], -3))), axis=1) if idx == 0: test_ind = regressor.predict_proba(Mahalanobis_test)[:, 1] test_ood_1 = regressor.predict_proba(Mahalanobis_ood_1)[:, 1] test_ood_2 = regressor.predict_proba(Mahalanobis_ood_2)[:, 1] test_ood_3 = regressor.predict_proba(Mahalanobis_ood_3)[:, 1] else: test_ind += regressor.predict_proba(Mahalanobis_test)[:, 1] test_ood_1 += regressor.predict_proba(Mahalanobis_ood_1)[:, 1] test_ood_2 += regressor.predict_proba(Mahalanobis_ood_2)[:, 1] test_ood_3 += regressor.predict_proba(Mahalanobis_ood_3)[:, 1] idx += 1 test_ind /= idx test_ood_1 /= idx test_ood_2 /= idx test_ood_3 /= idx return best_val_ind, best_val_ood, test_ind, test_ood_1, test_ood_2, test_ood_3