def evaluate(dataloader, model, criterion, accuracy, static_augmentations=[], device=None, random_seed=123): print("evaluating...") if random_seed is not None: utils.make_deterministic(random_seed) if device is None: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) model.eval() losses = AverageMeter() accs = [] nway = dataloader.batch_sampler.n_way nshot = dataloader.batch_sampler.n_shot nquery = dataloader.batch_sampler.n_query with torch.no_grad(): for i, data in enumerate(tqdm(dataloader)): inputs = data["input"].to(device) labels = data["label"].to(device) inputs_generated = None if model.mixer is not None or "generated" in static_augmentations: inputs_generated = data["generated"].to(device) print_final_nshot = False if i == 0: print_final_nshot = True outputs, query_labels = model(inputs, labels, nway, nshot, nquery, inputs_generated=inputs_generated, print_final_nshot=print_final_nshot, augmentations=static_augmentations) loss = criterion(outputs, query_labels) acc = accuracy(outputs, query_labels) losses.update(loss.item(), outputs.size(0)) accs.append(acc.item()) print("eval loss: %0.5f " % losses.avg) acc = float(np.mean(accs)) conf = float(1.96 * np.std(accs) / np.sqrt(len(accs))) print("eval acc :%0.5f +- %0.5f" % (acc, conf)) return float(losses.avg), acc, conf
def train_one_epoch(dataloader, model, criterion, optimizer, accuracy=accuracy, device=None, print_freq=100): since = time.time() if device is None: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) model.train() # Set model to training mode losses = AverageMeter() accs = AverageMeter() for i, data in enumerate(tqdm(dataloader)): inputs = data["input"].to(device) labels = data["label"].to(device) outputs = model(inputs) loss = criterion(outputs, labels) acc = accuracy(outputs, labels) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() losses.update(loss.item(), outputs.size(0)) accs.update(acc.item(), outputs.size(0)) if i % print_freq == 0 or i == len(dataloader) - 1: temp = "current loss: %0.5f " % loss.item() temp += "acc %0.5f " % acc.item() temp += "| running average loss %0.5f " % losses.avg temp += "acc %0.5f " % accs.avg print(i, temp) time_elapsed = time.time() - since print('this epoch took {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60)) return float(losses.avg), float(accs.avg)
def evaluate(dataloader, model, criterion, accuracy, device=None): if device is None: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) model.eval() losses = AverageMeter() accs = AverageMeter() with torch.no_grad(): for i, data in enumerate(tqdm(dataloader)): inputs = data["input"].to(device) labels = data["label"].to(device) outputs = model(inputs) loss = criterion(outputs, labels) acc = accuracy(outputs, labels) losses.update(loss.item(), outputs.size(0)) accs.update(acc.item(), outputs.size(0)) print("eval loss %0.5f acc %0.5f " % (losses.avg, accs.avg)) return float(losses.avg), float(accs.avg)
def main(args): device = gpu_setup(args.gpu) append_args = ["dataset", "model"] checkpoint_dir = savedir_setup(args.savedir, args=args, append_args=append_args, basedir=args.saveroot) args.githash = check_githash() save_args(checkpoint_dir, args) dataloader = setup_dataloader( name=args.dataset, batch_size=args.batch, num_workers=args.workers, ) dataset_size = len(dataloader.dataset) print("number of images (dataset size): ", dataset_size) model = setup_model(args.model, dataset_size=dataset_size, resume=args.resume, biggan_imagenet_pretrained_model_path=args.pretrained) model.eval() #this has to be eval() even if it's training time #because we want to fix batchnorm running mean and var #still tune batchnrom scale and bias that is generated by linear layer in biggan optimizer, scheduler = setup_optimizer( model, lr_g_linear=args.lr_g_l, lr_g_batch_stat=args.lr_g_batch_stat, lr_bsa_linear=args.lr_bsa_l, lr_embed=args.lr_embed, lr_class_cond_embed=args.lr_c_embed, step=args.step, step_facter=args.step_facter, ) criterion = AdaBIGGANLoss( scale_per=args.loss_per, scale_emd=args.loss_emd, scale_reg=args.loss_re, normalize_img=args.loss_norm_img, normalize_per=args.loss_norm_per, dist_per=args.loss_dist_per, ) #start trainig loop losses = AverageMeter() print_freq = args.print_freq eval_freq = args.eval_freq save_freq = eval_freq max_iteration = args.iters log = {} log["log"] = [] since = time.time() iteration = 0 epoch = 0 #prepare model and loss into device model = model.to(device) criterion = criterion.to(device) while (True): # Iterate over dataset (one epoch). for data in dataloader: img = data[0].to(device) indices = data[1].to(device) scheduler.step() #embeddings (i.e. z) + noise (i.e. epsilon) embeddings = model.embeddings(indices) embeddings_eps = torch.randn(embeddings.size(), device=device) * 0.01 #see https://github.com/nogu-atsu/SmallGAN/blob/f604cd17516963d8eec292f3faddd70c227b609a/gen_models/ada_generator.py#L29 #forward img_generated = model(embeddings + embeddings_eps) loss = criterion(img_generated, img, embeddings, model.linear.weight) losses.update(loss.item(), img.size(0)) #compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() if iteration % print_freq == 0: temp = "train loss: %0.5f " % loss.item() temp += "| smoothed loss %0.5f " % losses.avg log["log"].append({ "iteration": iteration, "epoch": epoch, "loss": losses.avg }) print(iteration, temp) losses = AverageMeter() if iteration % eval_freq == 0 and iteration > 0: out_path = os.path.join(checkpoint_dir, "%d_recon.jpg" % iteration) generate_samples(model, out_path, dataloader.batch_size) if iteration % save_freq == 0 and iteration > 0: save_checkpoint(checkpoint_dir, device, model, iteration=iteration) if iteration > max_iteration: break iteration += 1 if iteration > max_iteration: break epoch += 1 log_save_path = os.path.join(checkpoint_dir, "train-log.json") save_json(log, log_save_path)
def main(args): device = gpu_setup(args.gpu) append_args = ["dataset", "data_num", "model"] if args.mode == 'train': checkpoint_dir = savedir_setup(args.savedir, args=args, append_args=append_args, basedir=args.saveroot) args.githash = check_githash() save_args(checkpoint_dir, args) dataloader = setup_dataloader( name=args.dataset, batch_size=args.batch, num_workers=args.workers, data_num=args.data_num, ) dataset_size = len(dataloader.dataset) print("number of images (dataset size): ", dataset_size) if args.model == "biggan128-ada": model = setup_model( args.model, dataset_size=dataset_size, resume=args.resume, biggan_imagenet_pretrained_model_path=args.pretrained) elif args.model == "biggan128-conv1x1": model = setup_model( args.model, dataset_size=dataset_size, resume=args.resume, biggan_imagenet_pretrained_model_path=args.pretrained) elif args.model == 'biggan128-conv1x1-2' or args.model == 'biggan128-conv1x1-3': model = setup_model( args.model, dataset_size=dataset_size, resume=args.resume, biggan_imagenet_pretrained_model_path=args.pretrained, per_groups=args.per_groups) else: print('Error: Model not defined') sys.exit(1) model.eval() #this has to be eval() even if it's training time #because we want to fix batchnorm running mean and var #still tune batchnrom scale and bias that is generated by linear layer in biggan optimizer, scheduler = setup_optimizer( model_name=args.model, model=model, lr_g_linear=args.lr_g_l, lr_g_batch_stat=args.lr_g_batch_stat, lr_bsa_linear=args.lr_bsa_l, lr_embed=args.lr_embed, lr_class_cond_embed=args.lr_c_embed, step=args.step, step_facter=args.step_facter, ) criterion = AdaBIGGANLoss( scale_per=args.loss_per, scale_emd=args.loss_emd, scale_reg=args.loss_re, normalize_img=args.loss_norm_img, normalize_per=args.loss_norm_per, dist_per=args.loss_dist_per, ) #start trainig loop losses = AverageMeter() eval_kmmd = AverageMeter() print_freq = args.print_freq eval_freq = args.eval_freq save_freq = eval_freq max_epoch = args.epoch log = {} log["log"] = [] since = time.time() min_loss = { 'data': 1000, 'epoch': 0, } epoch = 0 #prepare model and loss into device model = model.to(device) criterion = criterion.to(device) while (True): if args.mode == 'train': scheduler.step() # Iterate over dataset (one epoch). for data in dataloader: define_seed(SEED) img = data[0].to(device) indices = data[1].to(device) #embeddings (i.e. z) + noise (i.e. epsilon) embeddings = model.embeddings(indices) embeddings_eps = torch.randn(embeddings.size(), device=device) * 0.01 #see https://github.com/nogu-atsu/SmallGAN/blob/f604cd17516963d8eec292f3faddd70c227b609a/gen_models/ada_generator.py#L29 #forward img_generated = model(embeddings + embeddings_eps) if args.mode == 'train': loss = criterion(img_generated, img, embeddings, model.linear.weight) losses.update(loss.item(), img.size(0)) #compute gradient and do SGD step # optimizer.zero_grad() # loss.backward() # optimizer.step() elif args.mode == 'eval': if args.KMMD: # KMMD latent_size = embeddings.size(1) true_sample = torch.randn(args.batch, latent_size, requires_grad=False).to(device) kmmd = KMMD()(true_sample, embeddings) eval_kmmd.update(kmmd.item(), img.size(0)) if args.mode == 'train': optimizer.zero_grad() loss.backward() optimizer.step() if epoch % print_freq == 0: if min_loss['data'] > losses.avg: min_loss['data'] = losses.avg min_loss['epoch'] = epoch temp = "train loss: %0.5f " % loss.item() temp += "| smoothed loss %0.5f " % losses.avg temp += "| min loss %0.5f (%d)" % (min_loss['data'], min_loss['epoch']) log["log"].append({"epoch": epoch, "loss": losses.avg}) print(epoch, temp) losses = AverageMeter() if epoch % eval_freq == 0: img_prefix = os.path.join(checkpoint_dir, "%d_" % epoch) generate_samples(model, img_prefix, dataloader.batch_size) if epoch % save_freq == 0: save_checkpoint(checkpoint_dir, device, model, iteration=epoch) elif args.mode == 'eval': if epoch * args.data_num > 10000: # eval用のデータ数は10,000 # if epoch * args.batch > 10000: # eval用のデータ数は10,000 if args.KMMD: print('KMMD:', eval_kmmd.avg) if args.FID: out_path = "./outputs/" + args.resume.split('/')[2] + '/' os.mkdir(out_path) for i in range(1000): visualizers.random_eval(model, out_path, tmp=0.3, n=1, truncate=True, roop_n=i) break if epoch > max_epoch: break epoch += 1 if args.mode == 'train': log_save_path = os.path.join(checkpoint_dir, "train-log.json") save_json(log, log_save_path)
def train_one_epoch(dataloader, model, criterion, optimizer, accuracy=accuracy, device=None, print_freq=100, random_seed=None): if random_seed is not None: #be careful to use this! #it's okay to fix seed every time we call evaluate() because we want to have exactly same order of test images #HOWEVER, for training time, we want to have different orders of training images for each epoch. #to do this, we can set the seed as epoch, for example. utils.make_deterministic(random_seed) since = time.time() if device is None: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) model.train() # Set model to training mode losses = AverageMeter() accs = AverageMeter() suprevised_baseline = False if hasattr(dataloader.batch_sampler, "n_way"): nway = dataloader.batch_sampler.n_way nshot = dataloader.batch_sampler.n_shot nquery = dataloader.batch_sampler.n_query else: suprevised_baseline = True for i, data in enumerate(tqdm(dataloader)): inputs = data["input"].to(device) labels = data["label"].to(device) if suprevised_baseline: #this is a baseline without meta-learning inputs = model.embed_samples(inputs) outputs = model.classifier(inputs) query_labels = labels else: inputs_generated = None if model.mixer is not None: inputs_generated = data["generated"].to(device) print_final_nshot = False if i == 0: print_final_nshot = True outputs, query_labels = model(inputs, labels, nway, nshot, nquery, inputs_generated=inputs_generated, print_final_nshot=print_final_nshot) loss = criterion(outputs, query_labels) acc = accuracy(outputs, query_labels) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure acc and record loss losses.update(loss.item(), outputs.size(0)) accs.update(acc.item(), outputs.size(0)) if i % print_freq == 0 or i == len(dataloader) - 1: temp = "current loss: %0.5f " % loss.item() temp += "acc %0.5f " % acc.item() temp += "| running average loss %0.5f " % losses.avg temp += "acc %0.5f " % accs.avg print(i, temp) time_elapsed = time.time() - since print('this epoch took {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60)) return float(losses.avg), float(accs.avg)