def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) for i, (name, module) in enumerate(model._modules.items()): module = recursion_change_bn(model) for m in model.modules(): if 'Conv' in str(type(m)): setattr(m, 'padding_mode', 'zeros') in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' % (args.net_type, args.dataset, 'FGSM')) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('\nNoise: ' + str(magnitude)) for i in range(num_output): M_in \ = lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out \ = lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy \ = lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, 'FGSM')) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 if args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 ################################ edits print("Calculate SVD before getting sample mean and variance") # svd_result= lib_generation.get_pca(model, args.num_classes, feature_list, train_loader) # lib_generation.get_pca_incremental(model, args.num_classes, feature_list, train_loader,args) svd_result = None print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader, svd_result, args) ################################ edits_end_sample_generator print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \ True, args.net_type, sample_mean, precision, i, magnitude,svd_result,args) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print('Out-distribution: ' + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \ False, args.net_type, sample_mean, precision, i, magnitude,svd_result,args) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, out_dist)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'malaria': args.num_classes = 2 if args.dataset == 'cifar100': args.num_classes = 100 # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) elif args.net_type == 'resnet18': model = models.get_model(args.net_type) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) model.cuda() print('load model: ' + args.net_type) # load dataset # selected_list = torch.LongTensor(selected_list) # test_clean_data = torch.index_select(test_clean_data, 0, selected_list) # test_adv_data = torch.index_select(test_adv_data, 0, selected_list) # test_noisy_data = torch.index_select(test_noisy_data, 0, selected_list) # test_label = torch.index_select(test_label, 0, selected_list) save_path = '%s/%s_%s/' % (args.outf, args.net_type, args.dataset) os.makedirs(save_path, exist_ok=True) # torch.save(test_clean_data, '%s/clean_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_adv_data, '%s/adv_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_noisy_data, '%s/noisy_data_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) # torch.save(test_label, '%s/label_%s_%s_%s.pth' % (save_path, args.model, args.dataset, attack)) print('load target data: ', args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, None, args.dataroot) test_clean_data = torch.load(args.outf + 'clean_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_adv_data = torch.load(args.outf + 'adv_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_noisy_data = torch.load(args.outf + 'noisy_data_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) test_label = torch.load(args.outf + 'label_%s_%s_%s.pth' % (args.net_type, args.dataset, args.adv_type)) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get LID scores') LID, LID_adv, LID_noisy \ = lib_generation.get_LID(model, test_clean_data, test_adv_data, test_noisy_data, test_label, num_output) overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90] list_counter = 0 for overlap in overlap_list: Save_LID = np.asarray(LID[list_counter], dtype=np.float32) Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32) Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32) Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy)) LID_data, LID_labels = lib_generation.merge_and_generate_labels( Save_LID_adv, Save_LID_pos) file_name = os.path.join( args.outf, 'LID_%s_%s_%s.npy' % (overlap, args.dataset, args.adv_type)) LID_data = np.concatenate((LID_data, LID_labels), axis=1) np.save(file_name, LID_data) list_counter += 1 print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('\nNoise: ' + str(magnitude)) for i in range(num_output): M_in \ = lib_generation.get_Mahalanobis_score_adv(model, test_clean_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out \ = lib_generation.get_Mahalanobis_score_adv(model, test_adv_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy \ = lib_generation.get_Mahalanobis_score_adv(model, test_noisy_data, test_label, \ args.num_classes, args.outf, args.net_type, \ sample_mean, precision, i, magnitude) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, args.adv_type)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth" args.outf = args.outf + args.net_type + "_" + args.dataset + "/" if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == "cifar100": args.num_classes = 100 if args.dataset == "svhn": out_dist_list = ["cifar10", "imagenet_resize", "lsun_resize"] else: out_dist_list = ["svhn", "imagenet_resize", "lsun_resize"] # load networks if args.net_type == "densenet": if args.dataset == "svhn": model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0), ), ]) elif args.net_type == "resnet": model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print("load model: " + args.net_type) # load dataset print("load target data: ", args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print("get sample mean and covariance") sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print("get Mahalanobis scores") m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print("Noise: " + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score( model, test_loader, args.num_classes, args.outf, True, args.net_type, sample_mean, precision, i, magnitude, ) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print("Out-distribution: " + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score( model, out_test_loader, args.num_classes, args.outf, False, args.net_type, sample_mean, precision, i, magnitude, ) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) ( Mahalanobis_data, Mahalanobis_labels, ) = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, "Mahalanobis_%s_%s_%s.npy" % (str(magnitude), args.dataset, out_dist), ) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): if args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] outf = os.path.join(args.outf, args.net_type + '_' + args.dataset + 'RealNVP') if os.path.isdir(outf) == False: os.mkdir(outf) # torch.cuda.manual_seed(0) torch.cuda.set_device(args.cuda_index) if args.dataset == 'cifar100': args.num_classes = 100 else: args.num_classes = 10 with open( os.path.join( 'feature_lists', 'feature_lists_{}_imagenet_resize_{}_Wlinear.pickle'.format( args.net_type, args.dataset)), 'rb') as f: [ sample_class_mean, list_features, list_features_test, list_features_out, A, A_inv, log_abs_det_A_inv ] = pickle.load(f) num_validation = 10000 v_size_per_class = int(num_validation / args.num_classes) X = [] X_validation = [] for i in range(args.num_classes): X_validation.append( list_features[args.layer][i][:v_size_per_class, :] - sample_class_mean[args.layer][i]) X.append(list_features[args.layer][i][v_size_per_class:, :] - sample_class_mean[args.layer][i]) train_loader_X = [] validation_loader_X = [] X_all = 0 for i in range(args.num_classes): if i == 0: X_all = X[i] else: X_all = torch.cat((X_all, X[i]), 0) for i in range(args.num_classes): train_loader_X.append( torch.utils.data.DataLoader(X[i], batch_size=args.batch_size, shuffle=True)) validation_loader_X.append( torch.utils.data.DataLoader(X_validation[i], batch_size=args.batch_size, shuffle=True)) num_features = A_inv[args.layer].shape[0] num_train = X_all.size(0) in_features_validation_test = list_features_test[args.layer] in_features_validation = in_features_validation_test[:2000, :] out_features_validation_test = list_features_out[args.layer] out_features_validation = out_features_validation_test[:2000, :] features_validation = torch.cat( (in_features_validation, out_features_validation), dim=0) label_validation = torch.cat( (torch.cuda.FloatTensor(in_features_validation.size(0)).fill_(1), torch.cuda.FloatTensor(out_features_validation.size(0)).fill_(0))) half_features = int(num_features / 2) zeros = np.zeros(half_features) ones = np.ones(half_features) right = np.concatenate((zeros, ones), axis=None) left = np.concatenate((ones, zeros), axis=None) masks = torch.from_numpy( np.array( [right, left, right, left, right, left, right, left, right, left]).astype(np.float32)).cuda() flow = [] optimizer = [] A_layer = torch.tensor(A[args.layer]) A_inv_layer = torch.tensor(A_inv[args.layer]) log_abs_det_A_inv_layer = torch.tensor(log_abs_det_A_inv[args.layer]) t_start = 0 for i in range(args.num_classes): flow.append( RealNVP(masks, num_features, args.length_hidden, A_layer, A_inv_layer, log_abs_det_A_inv_layer)) optimizer.append( torch.optim.Adam( [p for p in flow[i].parameters() if p.requires_grad == True], lr=args.lr)) sample_class_mean_layer = sample_class_mean[args.layer] loss_vec, auroc_vec, validation_vec = train(train_loader_X, flow, optimizer, args.num_iter, num_train, outf, args.layer, \ args.length_hidden, validation_loader_X, num_validation, features_validation, label_validation, sample_class_mean_layer,\ args.num_classes, args.dataset, t_start) features_test = list_features_test[args.layer] test_loader = torch.utils.data.DataLoader(features_test, batch_size=args.batch_size, shuffle=False) score_in = test(test_loader, flow, sample_class_mean_layer, args.num_classes) for out_dist in out_dist_list: with open( 'feature_lists/feature_lists_{}_{}_{}_Wlinear.pickle'.format( args.net_type, out_dist, args.dataset), 'rb') as f: [_, _, list_features_test, list_features_out, _, _, _] = pickle.load(f) if out_dist == 'FGSM': features_test = list_features_test[args.layer] test_loader = torch.utils.data.DataLoader( features_test, batch_size=args.batch_size, shuffle=False) score_in = test(test_loader, flow, sample_class_mean_layer, args.num_classes) features_out = list_features_out[args.layer] out_loader = torch.utils.data.DataLoader(features_out, batch_size=args.batch_size, shuffle=False) score_out = test(out_loader, flow, sample_class_mean_layer, args.num_classes) pram = { 'out_dist': out_dist, 'Network_type': args.net_type, 'Layer': args.layer, 'Batch_size': args.batch_size, 'num_iter': args.num_iter, 'cuda_index': args.cuda_index, 'lr': args.lr, 'lr_schedule': { 1: args.lr }, 'length_hidden': args.length_hidden, 'dropout': False, 'weight_decay': 0, 'init_zeros': True, 'num_flows': int(len(flow[0].t)), } with open( os.path.join( outf, 'Residual_flow_%s_%s_layer_%s_%siter_%sflows_%slength_hidden.txt' % (args.dataset, args.layer, args.num_iter, int(len(flow[0].t)), args.length_hidden)), 'w') as file: file.write('date: %s\n' % (datetime.datetime.now())) file.write(json.dumps(pram)) score_in = np.asarray(score_in, dtype=np.float32) score_out = np.asarray(score_out, dtype=np.float32) score_data, score_labels = lib_generation.merge_and_generate_labels( score_out, score_in) file_name = os.path.join( outf, 'Residual_flow_%s_%s_layer_%s_%siter_%sflows_%slength_hidden' % (args.dataset, out_dist, args.layer, args.num_iter, int(len(flow[0].t)), args.length_hidden)) score_data = np.concatenate((score_data, score_labels), axis=1) np.savez(file_name, score_data, loss_vec, pram, auroc_vec, validation_vec) std_Z_max = 0 std_Z_min = 1 for i in range(args.num_classes): std_Z = flow[i].f(X[i], training=False)[0].std(0) std_Z_max = max(std_Z_max, std_Z.max()) std_Z_min = min(std_Z_min, std_Z.min()) print('std z: \n', std_Z) print('std z max: \n', std_Z_max) print('std z min: \n', std_Z_min)
def extract_features(pre_trained_net, in_distribution, in_dist_name, out_dist_list, out_of_distribution, in_transform, gpu, batch_size, num_classes): # set the path to pre-trained model and output outf = "/output/" outf = outf + "model" + '_' + in_dist_name + '/' if os.path.isdir(outf) == False: os.mkdir(outf) torch.cuda.manual_seed(0) torch.cuda.set_device(gpu) # load networks model = torch.load(pre_trained_net, map_location="cuda:" + str(gpu)) model.cuda() print('loaded model') # load target dataset validation_split = .2 shuffle_dataset = True random_seed = 42 # Creating data indices for training and validation splits: dataset_size = len(in_distribution) indices = list(range(dataset_size)) split = int(np.floor(validation_split * dataset_size)) if shuffle_dataset: np.random.seed(random_seed) np.random.shuffle(indices) train_indices, val_indices = indices[split:], indices[:split] # Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) if in_transform is not None: train_loader = torch.utils.data.DataLoader(in_distribution, batch_size=batch_size, shuffle=True, sampler=train_sampler, **kwargs) test_loader = torch.utils.data.DataLoader(in_distribution, batch_size=batch_size, shuffle=True, sampler=valid_sampler, **kwargs) else: train_loader = torch.utils.data.DataLoader(in_distribution, batch_size=batch_size, shuffle=True, sampler=train_sampler, **kwargs) test_loader = torch.utils.data.DataLoader(in_distribution, batch_size=batch_size, shuffle=True, sampler=valid_sampler, **kwargs) print('loaded target data: ', in_dist_name) # set information about feature extaction model.eval() temp_x = torch.rand(*(list(next(iter(train_loader))[0].size()))).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, num_classes, feature_list, train_loader, model_name="model") print('Generate dataloaders...') out_test_loaders = [] for out_dist in out_of_distribution: out_test_loaders.append( torch.utils.data.DataLoader(out_dist, batch_size=batch_size, shuffle=True, **kwargs)) print('get Mahalanobis scores', num_output) m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): print('layer_num', i) M_in = lib_generation.get_Mahalanobis_score(model, test_loader, num_classes, outf, \ True, "model", sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_test_loader, out_dist in zip(out_test_loaders, out_dist_list): print('Out-distribution: ' + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, num_classes, outf, \ False, "model", sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), in_dist_name, out_dist)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): if args.validation_src == 'FGSM': if args.dataset == 'svhn': out_dist_list = [ 'cifar10', 'imagenet_resize', 'lsun_resize', 'FGSM' ] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize', 'FGSM'] else: if args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] else: out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] outf_load = os.path.join(args.outf, args.net_type + '_' + args.dataset + 'RealNVP') outf = os.path.join( args.outf, args.net_type + '_' + args.dataset + 'RealNVP_magnitude') if os.path.isdir(outf) == False: os.mkdir(outf) # torch.cuda.manual_seed(0) torch.cuda.set_device(args.cuda_index) if args.dataset == 'cifar100': args.num_classes = 100 else: args.num_classes = 10 with open( 'feature_lists/feature_lists_{}_imagenet_resize_{}_Wlinear.pickle'. format(args.net_type, args.dataset), 'rb') as f: [ sample_class_mean, list_features, list_features_test, list_features_out, A, A_inv, log_abs_det_A_inv ] = pickle.load(f) pre_trained_net = args.net_type + '_' + args.dataset + '.pth' pre_trained_net = os.path.join('pre_trained', pre_trained_net) if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.cuda_index))) else: model = torch.load(pre_trained_net, map_location="cpu") for i, (name, module) in enumerate(model._modules.items()): module = recursion_change_bn(model) for m in model.modules(): if 'Conv' in str(type(m)): setattr(m, 'padding_mode', 'zeros') in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) else: model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.cuda_index))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) print('load model: ' + args.net_type) model.to(device) model.eval() # load dataset print('load in-data: ', args.dataset) num_layers = len(sample_class_mean) for layer in range(num_layers): num_features = A_inv[layer].shape[0] half_features = int(num_features / 2) zeros = np.zeros(half_features) ones = np.ones(half_features) right = np.concatenate((zeros, ones), axis=None) left = np.concatenate((ones, zeros), axis=None) masks = torch.from_numpy( np.array([ right, left, right, left, right, left, right, left, right, left ]).astype(np.float32)).cuda() flow = [] # We reduce the number of neurons in the hidden layers due to GPU memory limitations (11 GB in GTX 2080Ti) - comment out this line for larger GPU memory length_hidden = reture_length_hidden(layer) A_layer = torch.tensor(A[layer]) A_inv_layer = torch.tensor(A_inv[layer]) log_abs_det_A_inv_layer = torch.tensor(log_abs_det_A_inv[layer]) for i in range(args.num_classes): MODEL_FLOW = os.path.join( outf_load, 'model_{}_layer_{}_residual_flow_{}length_hidden'.format( args.dataset, layer, length_hidden), 'flow_{}'.format(i)) flow.append( RealNVP(masks, num_features, length_hidden, A_layer, A_inv_layer, log_abs_det_A_inv_layer)) flow[i].load_state_dict(torch.load(MODEL_FLOW, map_location="cuda:{}".format( args.cuda_index)), strict=False) flow[i].to(device) flow[i].eval() sample_class_mean_layer = sample_class_mean[layer] m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) _, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) score_in = lib_generation.get_resflow_score( test_loader, model, layer, args.num_classes, args.net_type, sample_class_mean_layer, flow, magnitude) for out_dist in out_dist_list: print('load out-data: ', out_dist) if out_dist == 'FGSM': test_loader, out_loader = data_loader.getFGSM( args.batch_size, args.dataset, args.net_type) score_in = lib_generation.get_resflow_score_FGSM( test_loader, model, layer, args.num_classes, args.net_type, sample_class_mean_layer, flow, magnitude) score_out = lib_generation.get_resflow_score_FGSM( out_loader, model, layer, args.num_classes, args.net_type, sample_class_mean_layer, flow, magnitude) else: out_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) score_out = lib_generation.get_resflow_score( out_loader, model, layer, args.num_classes, args.net_type, sample_class_mean_layer, flow, magnitude) pram = { 'out_dist': out_dist, 'Network_type': args.net_type, 'Layer': layer, 'Batch_size': args.batch_size, 'cuda_index': args.cuda_index, 'length_hidden': length_hidden, 'dropout': False, 'weight_decay': 0, 'init_zeros': True, 'num_flows': int(len(flow[0].t)), 'magnitude': magnitude, } with open( os.path.join( outf, 'Residual_flow_%s_%s_layer_%s_%smagnitude.txt' % (args.dataset, out_dist, layer, magnitude)), 'w') as file: file.write('date: %s\n' % (datetime.datetime.now())) file.write(json.dumps(pram)) score_in = np.asarray(score_in, dtype=np.float32) score_out = np.asarray(score_out, dtype=np.float32) score_data, score_labels = lib_generation.merge_and_generate_labels( score_out, score_in) file_name = os.path.join( outf, 'Residual_flow_%s_%s_layer_%s_%smagnitude' % (args.dataset, out_dist, layer, magnitude)) score_data = np.concatenate((score_data, score_labels), axis=1) np.savez(file_name, score_data, pram)
def main(): # set the path to pre-trained model and output pre_trained_net = './pre_trained/' + args.net_type + '_' + args.dataset + '.pth' args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == 'cifar100': args.num_classes = 100 out_dist_list = ['svhn', 'imagenet_resize', 'lsun_resize'] elif args.dataset == 'svhn': out_dist_list = ['cifar10', 'imagenet_resize', 'lsun_resize'] elif args.dataset == 'ham10000': #out_dist_list = ['cifar10', 'imagenet_resize', 'face', 'face_age', 'isic-2017', 'isic-2016'] #out_dist_list = ['cifar10', 'face', 'face_age', 'isic-2017', 'isic-2016'] #out_dist_list = ['cifar10', 'cifar100', 'svhn', 'imagenet_resize', 'lsun_resize', 'face', 'face_age', 'isic-2017', 'isic-2016'] out_dist_list = [ 'ham10000-avg-smoothing', 'ham10000-brightness', 'ham10000-contrast', 'ham10000-dilation', 'ham10000-erosion', 'ham10000-med-smoothing', 'ham10000-rotation', 'ham10000-shift' ] # load networks if args.net_type == 'densenet': if args.dataset == 'svhn': model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0)), ]) elif args.net_type == 'resnet': model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) elif args.net_type == 'densenet121': model = DenseNet121(num_classes=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)).state_dict()) in_transform = transforms.Compose([ transforms.Resize(224), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize((0.7630069, 0.5456578, 0.5700767), (0.14093237, 0.15263236, 0.17000099)) ]) model.cuda() print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get Mahalanobis scores', num_output) m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): print('layer_num', i) M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \ True, args.net_type, sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print('Out-distribution: ' + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \ False, args.net_type, sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, out_dist)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output pre_trained_net = "./pre_trained/" + args.net_type + "_" + args.dataset + ".pth" args.outf = args.outf + args.net_type + "_" + args.dataset + "/" if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) # check the in-distribution dataset if args.dataset == "cifar100": args.num_classes = 100 # load networks if args.net_type == "densenet": if args.dataset == "svhn": model = models.DenseNet3(100, int(args.num_classes)) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) else: model = torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu)) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize( (125.3 / 255, 123.0 / 255, 113.9 / 255), (63.0 / 255, 62.1 / 255.0, 66.7 / 255.0), ), ]) elif args.net_type == "resnet": model = models.ResNet34(num_c=args.num_classes) model.load_state_dict( torch.load(pre_trained_net, map_location="cuda:" + str(args.gpu))) in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) model.cuda() print("load model: " + args.net_type) # load dataset print("load target data: ", args.dataset) train_loader, _ = data_loader.getTargetDataSet(args.dataset, args.batch_size, in_transform, args.dataroot) test_clean_data = torch.load(args.outf + "clean_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_adv_data = torch.load(args.outf + "adv_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_noisy_data = torch.load(args.outf + "noisy_data_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) test_label = torch.load(args.outf + "label_%s_%s_%s.pth" % (args.net_type, args.dataset, args.adv_type)) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 32, 32).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print("get sample mean and covariance") sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print("get LID scores") LID, LID_adv, LID_noisy = lib_generation.get_LID(model, test_clean_data, test_adv_data, test_noisy_data, test_label, num_output) overlap_list = [10, 20, 30, 40, 50, 60, 70, 80, 90] list_counter = 0 for overlap in overlap_list: Save_LID = np.asarray(LID[list_counter], dtype=np.float32) Save_LID_adv = np.asarray(LID_adv[list_counter], dtype=np.float32) Save_LID_noisy = np.asarray(LID_noisy[list_counter], dtype=np.float32) Save_LID_pos = np.concatenate((Save_LID, Save_LID_noisy)) LID_data, LID_labels = lib_generation.merge_and_generate_labels( Save_LID_adv, Save_LID_pos) file_name = os.path.join( args.outf, "LID_%s_%s_%s.npy" % (overlap, args.dataset, args.adv_type)) LID_data = np.concatenate((LID_data, LID_labels), axis=1) np.save(file_name, LID_data) list_counter += 1 print("get Mahalanobis scores") m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print("\nNoise: " + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score_adv( model, test_clean_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score_adv( model, test_adv_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) for i in range(num_output): M_noisy = lib_generation.get_Mahalanobis_score_adv( model, test_noisy_data, test_label, args.num_classes, args.outf, args.net_type, sample_mean, precision, i, magnitude, ) M_noisy = np.asarray(M_noisy, dtype=np.float32) if i == 0: Mahalanobis_noisy = M_noisy.reshape((M_noisy.shape[0], -1)) else: Mahalanobis_noisy = np.concatenate( (Mahalanobis_noisy, M_noisy.reshape( (M_noisy.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_noisy = np.asarray(Mahalanobis_noisy, dtype=np.float32) Mahalanobis_pos = np.concatenate((Mahalanobis_in, Mahalanobis_noisy)) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_pos) file_name = os.path.join( args.outf, "Mahalanobis_%s_%s_%s.npy" % (str(magnitude), args.dataset, args.adv_type), ) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def main(): # set the path to pre-trained model and output args.outf = args.outf + args.net_type + '_' + args.dataset + '/' if os.path.isdir(args.outf) == False: os.mkdir(args.outf) torch.cuda.manual_seed(0) torch.cuda.set_device(args.gpu) out_dist_list = [ 'skin_cli', 'skin_derm', 'corrupted', 'corrupted_70', 'imgnet', 'nct', 'final_test' ] # load networks if args.net_type == 'densenet_121': model = densenet_121.Net(models.densenet121(pretrained=False), 8) ckpt = torch.load("../checkpoints/densenet-121/checkpoint.pth") model.load_state_dict(ckpt['model_state_dict']) model.eval() model.cuda() elif args.net_type == 'mobilenet': model = mobilenet.Net(models.mobilenet_v2(pretrained=False), 8) ckpt = torch.load("../checkpoints/mobilenet/checkpoint.pth") model.load_state_dict(ckpt['model_state_dict']) model.eval() model.cuda() print("Done!") elif args.net_type == 'resnet_50': model = resnet_50.Net(models.resnet50(pretrained=False), 8) ckpt = torch.load("../checkpoints/resnet-50/checkpoint.pth") model.load_state_dict(ckpt['model_state_dict']) model.eval() model.cuda() print("Done!") elif args.net_type == 'vgg_16': model = vgg_16.Net(models.vgg16_bn(pretrained=False), 8) ckpt = torch.load("../checkpoints/vgg-16/checkpoint.pth") model.load_state_dict(ckpt['model_state_dict']) model.eval() model.cuda() print("Done!") else: raise Exception(f"There is no net_type={args.net_type} available.") in_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) print('load model: ' + args.net_type) # load dataset print('load target data: ', args.dataset) train_loader, test_loader = data_loader.getTargetDataSet( args.dataset, args.batch_size, in_transform, args.dataroot) # set information about feature extaction model.eval() temp_x = torch.rand(2, 3, 224, 224).cuda() temp_x = Variable(temp_x) temp_list = model.feature_list(temp_x)[1] num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 print('get sample mean and covariance') sample_mean, precision = lib_generation.sample_estimator( model, args.num_classes, feature_list, train_loader) print('get Mahalanobis scores') m_list = [0.0, 0.01, 0.005, 0.002, 0.0014, 0.001, 0.0005] for magnitude in m_list: print('Noise: ' + str(magnitude)) for i in range(num_output): M_in = lib_generation.get_Mahalanobis_score(model, test_loader, args.num_classes, args.outf, \ True, args.net_type, sample_mean, precision, i, magnitude) M_in = np.asarray(M_in, dtype=np.float32) if i == 0: Mahalanobis_in = M_in.reshape((M_in.shape[0], -1)) else: Mahalanobis_in = np.concatenate( (Mahalanobis_in, M_in.reshape((M_in.shape[0], -1))), axis=1) for out_dist in out_dist_list: out_test_loader = data_loader.getNonTargetDataSet( out_dist, args.batch_size, in_transform, args.dataroot) print('Out-distribution: ' + out_dist) for i in range(num_output): M_out = lib_generation.get_Mahalanobis_score(model, out_test_loader, args.num_classes, args.outf, \ False, args.net_type, sample_mean, precision, i, magnitude) M_out = np.asarray(M_out, dtype=np.float32) if i == 0: Mahalanobis_out = M_out.reshape((M_out.shape[0], -1)) else: Mahalanobis_out = np.concatenate( (Mahalanobis_out, M_out.reshape((M_out.shape[0], -1))), axis=1) Mahalanobis_in = np.asarray(Mahalanobis_in, dtype=np.float32) Mahalanobis_out = np.asarray(Mahalanobis_out, dtype=np.float32) Mahalanobis_data, Mahalanobis_labels = lib_generation.merge_and_generate_labels( Mahalanobis_out, Mahalanobis_in) file_name = os.path.join( args.outf, 'Mahalanobis_%s_%s_%s.npy' % (str(magnitude), args.dataset, out_dist)) Mahalanobis_data = np.concatenate( (Mahalanobis_data, Mahalanobis_labels), axis=1) np.save(file_name, Mahalanobis_data)
def get_conf(config, device, model, in_transform, train_loader, test_loader): # get layer list layer_list = config['exp_params']['feature_layers'] # get feature list model.eval() input_dim = config['exp_params']['input_dim'] # 2 x 3 x 32 x 32 if config['exp_params']['dataset'] != 'toy_data': temp_x = torch.rand(2, input_dim[0], input_dim[1], input_dim[2]).to(device) else: temp_x = torch.rand(2, 2).to(device) _, temp_list = model.feature_list(temp_x, layer_list) # ASK : Why this round about way to get feature_list size? Using temp_x etc. # RESPONSE: number of layers num_output = len(temp_list) feature_list = np.empty(num_output) count = 0 for out in temp_list: feature_list[count] = out.size(1) count += 1 # m_list is the list of noise m_list = config['exp_params']['noise_params']['m_list'] # calculate confidence components to be sent to regressor regressor_features = config['exp_params']['regressor_features'] class_mean, class_precision, tied_precision, pca_list, knn_search_list, knn_mean, knn_precision = \ get_inputs_for_computing_regressor_feature(regressor_features, model, config, num_output, feature_list, layer_list, train_loader, device) print("For in-distribution: {}".format(config['exp_params']['dataset'])) init_reg_in = True for regressor_feature in regressor_features: # num_output is the number of layers for i in range(num_output): in_dist_input = get_features_for_regressor( regressor_feature, model, config, test_loader, config['exp_params']['dataset'], i, True, device, class_mean, class_precision, tied_precision, pca_list, knn_search_list, knn_mean, knn_precision) print("in_dist_input shape: ", in_dist_input.shape) in_dist_input = np.asarray(in_dist_input, dtype=np.float32) # ASK: what does score of regression mean? print("Mean score at layer {} for regression type {}: {}".format( i, regressor_feature, np.mean(in_dist_input))) if init_reg_in: regressor_in_dist_input = in_dist_input.reshape( (in_dist_input.shape[0], -1)) init_reg_in = False else: regressor_in_dist_input = np.concatenate( (regressor_in_dist_input, in_dist_input.reshape((in_dist_input.shape[0], -1))), axis=1) print("Out-distributions to test agains: ", config['model_params']['out_dist_list']) for out_dist in config['model_params']['out_dist_list']: print('Out-distribution: ' + out_dist) if out_dist == 'subset_cifar100': out_test_loader = data_loader.getNonTargetDataSet( out_dist, config['trainer_params']['batch_size'], in_transform, config['exp_params']['dataroot'], idx=config['model_params']['out_idx'], num_oods=config['model_params']['num_ood_samples']) else: out_test_loader = data_loader.getNonTargetDataSet( out_dist, config['trainer_params']['batch_size'], in_transform, config['exp_params']['dataroot']) init_reg_in = True for regressor_feature in regressor_features: # num_output is the number of layers for i in range(num_output): out_dist_input = get_features_for_regressor( regressor_feature, model, config, out_test_loader, out_dist, i, False, device, class_mean, class_precision, tied_precision, pca_list, knn_search_list, knn_mean, knn_precision) print("out_dist_input shape- ", out_dist_input.shape) out_dist_input = np.asarray(out_dist_input, dtype=np.float32) print( "Mean score at layer {} for regression type {}: {}".format( i, regressor_feature, np.mean(out_dist_input))) if init_reg_in: regressor_out_dist_input = out_dist_input.reshape( (out_dist_input.shape[0], -1)) init_reg_in = False else: regressor_out_dist_input = np.concatenate( (regressor_out_dist_input, out_dist_input.reshape( (out_dist_input.shape[0], -1))), axis=1) regressor_in_dist_input = np.asarray(regressor_in_dist_input, dtype=np.float32) regressor_out_dist_input = np.asarray(regressor_out_dist_input, dtype=np.float32) ood_output, Mahalanobis_labels = lib_generation.merge_and_generate_labels( regressor_out_dist_input, regressor_in_dist_input) file_name = os.path.join( config['logging_params']['outf'], 'Mahalanobis_%s_%s_%s.npy' % (str(m_list[0]), config['exp_params']['dataset'], out_dist)) ood_output = np.concatenate((ood_output, Mahalanobis_labels), axis=1) np.save(file_name, ood_output) return ood_output