def gen_training_accuracy(args): # load data and model params = utils.load_params(args.model_dir) ckpt_dir = os.path.join(args.model_dir, 'checkpoints') ckpt_paths = [int(e[11:-3]) for e in os.listdir(ckpt_dir) if e[-3:] == ".pt"] ckpt_paths = np.sort(ckpt_paths) # csv headers = ["epoch", "acc_train", "acc_test"] csv_path = utils.create_csv(args.model_dir, 'accuracy.csv', headers) for epoch, ckpt_paths in enumerate(ckpt_paths): if epoch % 5 != 0: continue net, epoch = tf.load_checkpoint(args.model_dir, epoch=epoch, eval_=True) # load data train_transforms = tf.load_transforms('test') trainset = tf.load_trainset(params['data'], train_transforms, train=True) trainloader = DataLoader(trainset, batch_size=500, num_workers=4) train_features, train_labels = tf.get_features(net, trainloader, verbose=False) test_transforms = tf.load_transforms('test') testset = tf.load_trainset(params['data'], test_transforms, train=False) testloader = DataLoader(testset, batch_size=500, num_workers=4) test_features, test_labels = tf.get_features(net, testloader, verbose=False) acc_train, acc_test = svm(args, train_features, train_labels, test_features, test_labels) utils.save_state(args.model_dir, epoch, acc_train, acc_test, filename='accuracy.csv') print("Finished generating accuracy.")
def gen_testloss(args): # load data and model params = utils.load_params(args.model_dir) ckpt_dir = os.path.join(args.model_dir, 'checkpoints') ckpt_paths = [int(e[11:-3]) for e in os.listdir(ckpt_dir) if e[-3:] == ".pt"] ckpt_paths = np.sort(ckpt_paths) # csv headers = ["epoch", "step", "loss", "discrimn_loss_e", "compress_loss_e", "discrimn_loss_t", "compress_loss_t"] csv_path = utils.create_csv(args.model_dir, 'losses_test.csv', headers) print('writing to:', csv_path) # load data test_transforms = tf.load_transforms('test') testset = tf.load_trainset(params['data'], test_transforms, train=False) testloader = DataLoader(testset, batch_size=params['bs'], shuffle=False, num_workers=4) # save loss criterion = MaximalCodingRateReduction(gam1=params['gam1'], gam2=params['gam2'], eps=params['eps']) for epoch, ckpt_path in enumerate(ckpt_paths): net, epoch = tf.load_checkpoint(args.model_dir, epoch=epoch, eval_=True) for step, (batch_imgs, batch_lbls) in enumerate(testloader): features = net(batch_imgs.cuda()) loss, loss_empi, loss_theo = criterion(features, batch_lbls, num_classes=len(testset.num_classes)) utils.save_state(args.model_dir, epoch, step, loss.item(), *loss_empi, *loss_theo, filename='losses_test.csv') print("Finished generating test loss.")
'--cpb', type=int, default=10, help='number of classes in each learning batch (default: 10)') parser.add_argument('--save', action='store_true', help='save labels') parser.add_argument('--data_dir', default='./data/', help='path to dataset') args = parser.parse_args() print("evaluate using label_batch: {}".format(args.label_batch)) params = utils.load_params(args.model_dir) # get train features and labels train_transforms = tf.load_transforms('test') trainset = tf.load_trainset(params['data'], train_transforms, train=True, path=args.data_dir) if 'lcr' in params.keys(): # supervised corruption case trainset = tf.corrupt_labels(trainset, params['lcr'], params['lcs']) new_labels = trainset.targets assert (trainset.num_classes % args.cpb == 0), "Number of classes not divisible by cpb" ## load model net, epoch = tf.load_checkpoint_ce(args.model_dir, trainset.num_classes, args.epoch, eval_=True, label_batch_id=args.label_batch)
## per model functions def lr_schedule(epoch, optimizer): """decrease the learning rate""" lr = args.lr if epoch >= 400: lr = args.lr * 0.01 elif epoch >= 200: lr = args.lr * 0.1 for param_group in optimizer.param_groups: param_group['lr'] = lr ## Prepare for Training transforms = tf.load_transforms(args.transform) trainset = tf.load_trainset(args.data, transforms, path=args.data_dir) #trainset = tf.corrupt_labels(trainset, args.lcr, args.lcs) if args.pretrain_dir is not None: net, _ = tf.load_checkpoint(args.pretrain_dir, args.pretrain_epo) utils.update_params(model_dir, args.pretrain_dir) else: net = tf.load_architectures_ce(args.arch, trainset.num_classes) assert (trainset.num_classes % args.cpb == 0), "Number of classes not divisible by cpb" classes = np.unique(trainset.targets) class_batch_num = trainset.num_classes // args.cpb class_batch_list = classes.reshape(class_batch_num, args.cpb) #trainloader = DataLoader(trainset, batch_size=args.bs, drop_last=True, num_workers=4) criterion = nn.CrossEntropyLoss()
def plot_pca_epoch(args): """Plot PCA for different epochs in the same plot. """ EPOCHS = [0, 10, 100, 500] params = utils.load_params(args.model_dir) transforms = tf.load_transforms('test') trainset = tf.load_trainset(params['data'], transforms) trainloader = DataLoader(trainset, batch_size=200, num_workers=4) sig_vals = [] for epoch in EPOCHS: epoch_ = epoch - 1 if epoch_ == -1: # randomly initialized net = tf.load_architectures(params['arch'], params['fd']) else: net, epoch = tf.load_checkpoint(args.model_dir, epoch=epoch_, eval_=True) features, labels = tf.get_features(net, trainloader) if args.class_ is not None: features_sort, _ = utils.sort_dataset( features.numpy(), labels.numpy(), num_classes=trainset.num_classes, stack=False) features_ = features_sort[args.class_] else: features_ = features.numpy() n_comp = np.min([args.comp, features.shape[1]]) pca = PCA(n_components=n_comp).fit(features_) sig_vals.append(pca.singular_values_) ## plot singular values plt.rc('text', usetex=True) plt.rcParams['font.family'] = 'serif' plt.rcParams['font.serif'] = ['Times New Roman'] fig, ax = plt.subplots(1, 1, figsize=(7, 5), dpi=400) x_min = np.min([len(sig_val) for sig_val in sig_vals]) if args.class_ is not None: ax.set_xticks(np.arange(0, x_min, 10)) ax.set_yticks(np.linspace(0, 40, 9)) ax.set_ylim(0, 40) else: ax.set_xticks(np.arange(0, x_min, 10)) ax.set_yticks(np.linspace(0, 80, 9)) ax.set_ylim(0, 90) for epoch, sig_val in zip(EPOCHS, sig_vals): ax.plot(np.arange(x_min), sig_val[:x_min], marker='', markersize=5, label=f'epoch - {epoch}', alpha=0.6) ax.legend(loc='upper right', frameon=True, fancybox=True, prop={"size": 8}, ncol=1, framealpha=0.5) ax.set_xlabel("components") ax.set_ylabel("sigular values") ax.spines['top'].set_visible(False) ax.spines['right'].set_visible(False) ax.spines['bottom'].set_visible(False) ax.spines['left'].set_visible(False) [tick.label.set_fontsize(12) for tick in ax.xaxis.get_major_ticks()] [tick.label.set_fontsize(12) for tick in ax.yaxis.get_major_ticks()] ax.grid(True, color='white') ax.set_facecolor('whitesmoke') fig.tight_layout() ## save save_dir = os.path.join(args.model_dir, 'figures', 'pca') np.save(os.path.join(save_dir, "sig_vals_epoch.npy"), sig_vals) if not os.path.exists(save_dir): os.makedirs(save_dir) file_name = os.path.join(save_dir, f"pca_class{args.class_}.png") fig.savefig(file_name) print("Plot saved to: {}".format(file_name)) file_name = os.path.join(save_dir, f"pca_class{args.class_}.pdf") fig.savefig(file_name) print("Plot saved to: {}".format(file_name)) plt.close()
if args.traintest: path = os.path.join(args.model_dir, 'losses_test.csv') if not os.path.exists(path): gen_testloss(args) plot_traintest(args, path) if args.acc: path = os.path.join(args.model_dir, 'accuracy.csv') if not os.path.exists(path): gen_training_accuracy(args) plot_accuracy(args, path) if args.pca or args.hist or args.heat or args.nearcomp_sup or args.nearcomp_unsup or args.nearcomp_class: ## load data and model params = utils.load_params(args.model_dir) net, epoch = tf.load_checkpoint(args.model_dir, args.epoch, eval_=True) transforms = tf.load_transforms('test') trainset = tf.load_trainset(params['data'], transforms) if 'lcr' in params.keys(): # supervised corruption case trainset = tf.corrupt_labels(trainset, params['lcr'], params['lcs']) trainloader = DataLoader(trainset, batch_size=200, num_workers=4) features, labels = tf.get_features(net, trainloader) if args.pca: plot_pca(args, features, labels, epoch) if args.nearcomp_sup: plot_nearest_component_supervised(args, features, labels, epoch, trainset) if args.nearcomp_unsup: plot_nearest_component_unsupervised(args, features, labels, epoch, trainset)