def train_model(model, dataset, ds_name, epochs=10, batch_size=32, sample_size=32, eval_size=32, img_size=32, lr=1e-3, weight_decay=1e-4, loss_log_interval=20, image_log_interval=20, model_log_interval=20, checkpoint_dir='./checkpoints', resume=False, cuda=False, seed=0, device=None, cores=1): if resume: epoch_start = utils.load_checkpoint(model, checkpoint_dir) else: epoch_start = 0 fixed_noise = torch.rand(sample_size, model.z_size).to(device) if model.model_name in ['vae', 'cvae', 'vae2', 'cvae2']: m = dist.Normal(torch.Tensor([0.0]).to(device), torch.Tensor([1.0]).to(device)) fixed_noise = m.icdf(fixed_noise) output_folder = './results/' + ds_name resfile_prefix = ds_name + "_" + \ model.model_name + \ "_ld_" + \ str(model.z_size) + \ "_bs_" + str(batch_size) if not os.path.exists(output_folder): os.makedirs(output_folder) data_root = './datasets' if model.model_name in ['dec_vine', 'dec_vine2', 'dec_vine3']: # load pre-trained AE if model.model_name == 'dec_vine': pretrain_prefix = resfile_prefix.replace("dec", "ae") elif model.model_name == 'dec_vine2': pretrain_prefix = resfile_prefix.replace("dec_vine2", "ae_vine2") elif model.model_name == 'dec_vine3': pretrain_prefix = resfile_prefix.replace("dec_vine3", "ae_vine3") pretrain_files = [filename for filename in os.listdir(checkpoint_dir) if filename.startswith(pretrain_prefix)] pretrain_epochs = [int(filename.replace(pretrain_prefix + "_", "")) for filename in pretrain_files] pretrain_path = os.path.join(checkpoint_dir, pretrain_files[pretrain_epochs.index(max(pretrain_epochs))]) model.pretrain(pretrain_path) # form initial cluster centres data_loader = utils.get_data_loader(dataset, batch_size, cuda=cuda) data_stream = tqdm(enumerate(data_loader, 1)) features = [] for batch_index, (x, _, _) in data_stream: tmp_x = Variable(x).to(device) if model.model_name == 'dec_vine': z = model.ae.encoder(tmp_x) z = model.ae.q(z) elif model.model_name == 'dec_vine2' or model.model_name == 'dec_vine3': z = torch.nn.functional.relu(model.ae.fc1(model.ae.encoder(tmp_x).view(x.size(0), -1))) z = model.ae.fc21(z) features.append(z) kmeans = KMeans(n_clusters=model.cluster_number, n_init=20) y_pred = kmeans.fit_predict(torch.cat(features).detach().cpu().numpy()) model.cluster_layer.data = torch.tensor(kmeans.cluster_centers_).to(device) pretrain=0 if pretrain==1 and model.model_name == 'ae_vine3': pretrain_prefix = resfile_prefix#ds_name + '_ae_vine3' pretrain_files = [filename for filename in os.listdir(checkpoint_dir) if filename.startswith(pretrain_prefix)] pretrain_epochs = [int(filename.replace(pretrain_prefix + "_", "")) for filename in pretrain_files] pretrain_path = os.path.join(checkpoint_dir, pretrain_files[pretrain_epochs.index(max(pretrain_epochs))]) pretrained_ae = torch.load(pretrain_path, map_location=device) model.load_state_dict(pretrained_ae['state']) print('load pretrained ae3 from', pretrain_path) # reconstruction_criterion = torch.nn.BCELoss() reconstruction_criterion = torch.nn.BCELoss(size_average=False) optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) if model.model_name == 'gan': lr_g = lr_d = 0.0002 k = 1 fix_noise = get_noise(sample_size) opt_g = torch.optim.Adam(model.net_g.parameters(), lr=lr_g, betas=(0.5, 0.999)) # optimizer for Generator opt_d = torch.optim.Adam(model.net_d.parameters(), lr=lr_d, betas=(0.5, 0.999)) # optimizer for Discriminator for epoch in range(epoch_start, epochs + 1): print("Epoch {}".format(epoch)) if model.model_name == "dec_vine" or model.model_name == "dec_vine2": # update target distribution p model.eval() p = [] indices = [] data_loader = utils.get_data_loader(dataset, batch_size, cuda=cuda) data_stream = tqdm(enumerate(data_loader, 1)) for batch_index, (x, _, idx) in data_stream: tmp_x = Variable(x).to(device) _, tmp_p = model(tmp_x) p.append(tmp_p.detach().cpu()) tmp_idx = idx indices.append(tmp_idx) p = torch.cat(p) indices = torch.cat(indices) p = model.target_distribution(p[indices]) p = Variable(p).to(device) model.train() data_loader = utils.get_data_loader(dataset, batch_size, cuda=cuda) data_stream = tqdm(enumerate(data_loader, 1)) for batch_index, (x, _, idx) in data_stream: # learning rate decay if model.model_name == 'gan' and (epoch) == 8:# and dataset == "CelebA": opt_g.param_groups[0]['lr'] /= 10 opt_d.param_groups[0]['lr'] /= 10 #print("learning rate change!") if model.model_name == 'gan' and (epoch) == 15: # and dataset == "CelebA": opt_g.param_groups[0]['lr'] /= 10 opt_d.param_groups[0]['lr'] /= 10 #print("learning rate change!") iteration = (epoch - 1) * (len(dataset) // batch_size) + batch_index x = Variable(x).to(device) idx = Variable(idx).to(device) if model.model_name == 'gan': # train Discriminator real_data = Variable(x.cuda()) #print(real_data.shape) prob_fake = model.net_d(model.net_g(get_noise(real_data.size(0)).to(device))) prob_real = model.net_d(real_data) loss_d = - torch.mean(torch.log(prob_real) + torch.log(1 - prob_fake)) opt_d.zero_grad() loss_d.backward() opt_d.step() # train Generator if batch_index % k is 0: prob_fake = model.net_d(model.net_g(get_noise().to(device))) loss_g = - torch.mean(torch.log(prob_fake)) opt_g.zero_grad() loss_g.backward() opt_g.step() else: if model.model_name == 'ae_vine' or model.model_name == 'ae_vine2' or model.model_name == 'ae_vine3': x_reconstructed = model(x) elif model.model_name == 'dec_vine' or model.model_name == 'dec_vine2': x_reconstructed, q = model(x) p_batch = p[idx] penalization_loss = 10*F.kl_div(q.log(), p_batch) del p_batch, q elif model.model_name == 'cvae' or model.model_name == "cvae2" or model.model_name=="cvae3": (mean, logvar, atanhcor), x_reconstructed = model(x) penalization_loss = model.kl_divergence_loss(mean, logvar, atanhcor) elif model.model_name == 'vae' or model.model_name == "vae2" or model.model_name=="vae3": (mean, logvar), x_reconstructed = model(x) penalization_loss = model.kl_divergence_loss(mean, logvar) reconstruction_loss = reconstruction_criterion(x_reconstructed, x) / x.size(0) if model.model_name == 'ae_vine' or model.model_name == 'ae_vine2' or model.model_name == 'ae_vine3': loss = reconstruction_loss else: loss = reconstruction_loss + penalization_loss optimizer.zero_grad() loss.backward() optimizer.step() if iteration % loss_log_interval == 0: f = open(output_folder + "/" + resfile_prefix + "_losses" + ".txt", 'a') if model.model_name == 'gan': f.write("\n{:<12} | {} | {} | {} | {} ".format( model.model_name, iteration, loss_g, loss_d, seed )) ''' print("\n{:<12} | {} | {} | {} | {} ".format( model.model_name, iteration, loss_g, loss_d, seed )) ''' else: if model.model_name == 'ae_vine' or model.model_name == 'ae_vine2' or model.model_name == 'ae_vine3': f.write("\n{:<12} | {} | {} | {} ".format( model.model_name, iteration, loss, seed )) else: f.write("\n{:<12} | {} | {} | {} | {} | {}".format( model.model_name, iteration, reconstruction_loss.data.item(), penalization_loss.data.item(), loss.data.item(), seed )) f.close() # adding this just to have a way of calculating the scores at 0 epochs if batch_index > 0 and epoch == 0: break if epoch % model_log_interval == 0: print() print('###################') print('# model checkpoint!') print('###################') print() utils.save_checkpoint(model, checkpoint_dir, epoch, resfile_prefix + "_" + str(epoch)) if epoch % image_log_interval == 0: print() print('###################') print('# image checkpoint!') print('###################') print() model.eval() ae_vine_models = ['ae_vine', 'ae_vine2', 'dec_vine', 'dec_vine2', 'ae_vine3', 'dec_vine3'] if model.model_name in ae_vine_models: data_loader_vine = utils.get_data_loader(dataset, 5000, cuda=cuda) data_stream_vine = tqdm(enumerate(data_loader_vine, 1)) features = [] for batch_index, (x, _, _) in data_stream_vine: tmp_x = Variable(x).to(device) if model.model_name == 'ae_vine': encoded = model.encoder(tmp_x) e = model.q(encoded) elif model.model_name == 'dec_vine': encoded = model.ae.encoder(tmp_x) e = model.ae.q(encoded) elif model.model_name == 'ae_vine2': encoded = torch.nn.functional.relu(model.fc1(model.encoder(tmp_x).view(x.size(0), -1))) e = model.fc21(encoded) elif model.model_name == 'dec_vine2': encoded = torch.nn.functional.relu(model.ae.fc1(model.ae.encoder(tmp_x).view(x.size(0), -1))) e = model.ae.fc21(encoded) elif model.model_name == 'ae_vine3': encoded = F.relu(model.fc1(model.encoder(tmp_x).view(x.size(0), -1))) e = model.fc21(encoded) elif model.model_name == 'dec_vine3': encoded = F.relu(model.ae.fc1(model.ae.encoder(tmp_x).view(x.size(0), -1))) e = model.ae.fc21(encoded) features.append(e.detach().cpu()) if batch_index > 0: break features = torch.cat(features).numpy() #np.savetxt(resfile_prefix + '_features' + str(epoch) + '_.csv', features, delimiter=",") copula_controls = base.list(family_set="tll", trunc_lvl=5, cores=cores) vine_obj = rvinecop.vine(features, copula_controls=copula_controls) model.vine = vine_obj fake = model.sample(sample_size, vine_obj, fixed_noise) del x, e, encoded, vine_obj,data_loader_vine elif model.model_name == 'gan': fake = model.net_g(fix_noise.to(device)).data.cpu() #+ 0.5 print(fake.shape) else: fake = model.sample(sample_size, fixed_noise) fake = fake.reshape(sample_size, model.channel_num, model.image_size, model.image_size) name_str = resfile_prefix + '_fake_samples_epoch' vutils.save_image(fake.detach(), '%s/%s_%03d.png' % (output_folder, name_str, epoch), normalize=True) del fake if epoch % 10 == 0: s = metric.compute_score_raw(ds_name, dataset, img_size, data_root, eval_size, batch_size, output_folder + '/real/', output_folder + '/fake/', model, model.z_size, 'resnet34', device) f = open(output_folder + "/" + resfile_prefix + "_scores" + ".txt", 'a') scr_arr = [str(a) for a in s] f.write("\n{:<12} | {} | {} | {}".format( model.model_name, epoch, ', '.join(scr_arr), seed )) f.close()
criterion = nn.BCELoss() fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=device) real_label = 1 fake_label = 0 # setup optimizer optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) # [emd-mmd-knn(knn,real,fake,precision,recall)]*4 - IS - mode_score - FID score_tr = np.zeros((opt.niter, 4*7+3)) # compute initial score s = metric.compute_score_raw(opt.dataset, opt.imageSize, opt.dataroot, opt.sampleSize, 16, opt.outf+'/real/', opt.outf+'/fake/', netG, opt.nz, conv_model='inception_v3', workers=int(opt.workers)) score_tr[0] = s np.save('%s/score_tr.npy' % (opt.outf), score_tr) ######################### #### Models training #### ######################### for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): ############################ # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) ########################### # train with real netD.zero_grad() real_cpu = data[0].to(device) batch_size = real_cpu.size(0)
betas=(opt.beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) # [emd-mmd-knn(knn,real,fake,precision,recall)]*4 - IS - mode_score - FID score_tr = np.zeros((opt.niter, 4 * 7 + 3)) # compute initial score s = metric.compute_score_raw(opt.dataset, opt.imageSize, opt.dataroot, opt.sampleSize, 16, opt.outf + '/real/', opt.outf + '/fake/', netG, opt.nz, conv_model='inception_v3', workers=int(opt.workers)) score_tr[0] = s np.save('%s/score_tr.npy' % (opt.outf), score_tr) ######################### #### Models training #### ######################### for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): ############################ # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))
fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=device) real_label = 1 fake_label = 0 # setup optimizer optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) score_tr = np.zeros( (opt.niter, 4 * 7 + 3) ) #[emd-mmd-knn(knn,real,fake,precision,recall)]*4 - incep-mode_score-fid # compute training score s = metric.compute_score_raw(opt.dataset, opt.imageSize, opt.dataroot, opt.sampleSize, 50, opt.outf + '/real/', opt.outf + '/fake/', netG, opt.nz, 'inception_v3') score_tr[0] = s np.save('%s/score_tr.npy' % (opt.outf), score_tr) for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): ############################ # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) ########################### # train with real netD.zero_grad() real_cpu = data[0].to(device) batch_size = real_cpu.size(0) label = torch.full((batch_size, ), real_label, device=device)