# # --- train --- #
train_vars = [
'batch_size', 'D_rounds', 'G_rounds', 'use_time', 'seq_length',
'latent_dim', 'num_generated_features', 'cond_dim', 'max_val', 'WGAN_clip',
'one_hot'
]
train_settings = dict((k, settings[k]) for k in train_vars)
t0 = time()
best_epoch = 0
print('epoch\ttime\tD_loss\tG_loss\tmmd2\t')
for epoch in range(num_epochs):
D_loss_curr, G_loss_curr = model.train_epoch(epoch, samples['train'],
labels['train'], sess, Z, X,
CG, CD, CS, accuracy, D_loss,
G_loss, D_solver, G_solver,
**train_settings)
# -- eval -- #
# visualise plots of generated samples, with/without labels
if epoch % vis_freq == 0:
if CGAN:
vis_sample = sess.run(G_sample, feed_dict={Z: vis_Z, CG: vis_C})
else:
vis_sample = sess.run(G_sample, feed_dict={Z: vis_Z})
plotting.visualise_at_epoch(vis_sample,
data,
predict_labels,
one_hot,
epoch,
identifier,
train_loader = DataLoader(train_data,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
shuffle=True)
val_loader = DataLoader(val_data,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS)
device = torch.device('cuda')
model = model.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = Adam(model.parameters(), lr=LEARNING_RATE)
train_losses = []
val_losses = []
for epoch in range(NUM_EPOCHS):
train_loss = train_epoch(model, train_loader, criterion, optimizer,
device)
train_losses.append(train_loss)
message = f'Epoch: {epoch}\tTrainLoss: {train_loss}'
if len(
val_data
) > 0: # only run validation epoch if the validation dataset is not empty
val_loss, val_acc = val_epoch(model, val_loader, criterion, device)
val_losses.append(val_loss)
message += f'\tValLoss: {val_loss}\tValAcc: {val_acc}'
print(message)
if epoch % CHECKPOINT_RATE == 0:
print('Checkpointing model...')
checkpoint(model, os.path.join(MODEL_DIR, f'fer_model_{epoch}.pt'))
model_load(model.net_D, args.checkpoint_d)
model.to(device)
lr_scheduler_G = optim.lr_scheduler.StepLR(
model.optimizer_G, step_size=100, gamma=0.1)
if model.use_D:
lr_scheduler_D = optim.lr_scheduler.StepLR(
model.optimizer_D, step_size=100, gamma=0.1)
# get data loader
train_dl, valid_dl = get_data(trainning=True, bs=args.bs)
for epoch in range(args.epochs):
print("Epoch {}/{}, learning rate: {} ...".format(epoch + 1,
args.epochs, lr_scheduler_G.get_last_lr()))
train_epoch(train_dl, model, device, tag='train')
valid_epoch(valid_dl, model, device, tag='valid')
lr_scheduler_G.step()
if model.use_D:
lr_scheduler_D.step()
if (epoch + 1) % 100 == 0 or (epoch == args.epochs - 1):
model_save(model.net_G, os.path.join(
args.outputdir, "ImageColor_G_{}.pth".format(epoch + 1)))
if (model.use_D):
model_save(model.net_G, os.path.join(
args.outputdir, "ImageColor_D_{}.pth".format(epoch + 1)))
'batch_size', 'D_rounds', 'G_rounds', 'use_time', 'seq_length',
'latent_dim'
]
train_settings = dict((k, settings[k]) for k in train_vars)
train_settings['num_signals'] = num_variables
t0 = time()
MMD = np.zeros([
num_epochs,
])
for epoch in range(num_epochs):
# for epoch in range(1):
# -- train epoch -- #
D_loss_curr, G_loss_curr = model.train_epoch(epoch, samples, labels, sess,
Z, X, D_loss, G_loss,
D_solver, G_solver,
**train_settings)
# # -- eval -- #
# # visualise plots of generated samples, with/without labels
# # choose which epoch to visualize
#
# # random input vectors for the latent space, as the inputs of generator
# vis_ZZ = model.sample_Z(batch_size, seq_length, latent_dim, use_time)
#
# # # -- generate samples-- #
# vis_sample = sess.run(G_sample, feed_dict={Z: vis_ZZ})
# # # -- visualize the generated samples -- #
# plotting.save_plot_sample(vis_sample, epoch, identifier, n_samples=16, num_epochs=None, ncol=4)
# # plotting.save_plot_sample(vis_sample, 0, identifier + '_real', n_samples=16, num_epochs=num_epochs)
# # # save the generated samples in cased they might be useful for comparison
t0 = time()
best_epoch = 0
print('epoch\ttime\tD_loss\tG_loss\tmmd2\tpdf_sample\tpdf_real')
mmd_calc = None
kernel_calc = None
for epoch in range(num_epochs):
D_loss_curr, G_loss_curr = model.train_epoch(
epoch,
samples['train'],
labels['train'],
sess,
Z,
X,
CG,
CD,
CS,
D_loss,
G_loss,
#D_logit_real, D_logit_fake,
#conv, layer, w,
D_solver,
G_solver,
**train_settings)
# compute mmd2 and, if available, prob density
if epoch % eval_freq == 0:
## how many samples to evaluate with?
validation = np.float32(samples['vali'][np.random.choice(
len(samples['vali']), size=batch_multiplier * batch_size), :, :])
eval_Z = validation[:, :
optimizer = optim.SGD(params,
lr=args.lr,
momentum=0.9,
weight_decay=0.0005)
lr_scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=1000,
gamma=0.1)
# get data loader
train_dl, valid_dl = get_data(trainning=True, bs=args.bs)
for epoch in range(args.epochs):
print("Epoch {}/{}, learning rate: {} ...".format(
epoch + 1, args.epochs, lr_scheduler.get_last_lr()))
train_epoch(train_dl, model, optimizer, device, tag='train')
valid_epoch(valid_dl, model, device, tag='valid')
lr_scheduler.step()
#
# /************************************************************************************
# ***
# *** MS: Define Save Model Strategy
# ***
# ************************************************************************************/
#
if epoch == (args.epochs // 2) or (epoch == args.epochs - 1):
model_save(model,
os.path.join(args.outputdir, "latest-checkpoint.pth"))
train_dataloader = DataLoader(train_set, batch_size=BATCH_SIZE, collate_fn=data.train_create_batch, shuffle = True)
val_dataloader = DataLoader(val_set, batch_size=BATCH_SIZE, collate_fn=data.val_create_batch,shuffle = True)
test_dataloader = DataLoader(test_set, batch_size=BATCH_SIZE, collate_fn=data.val_create_batch,shuffle = True)
# writer.flush()
best_accuracy = 0
for epoch in range(EPOCHS):
print(f'Epoch {epoch + 1}/{EPOCHS}')
print('----------')
train_acc, train_loss, train_cfs_matrix = m.train_epoch(
model,
train_dataloader,
loss_function,
optimizer,
device,
output_type,
len(articles_train)
)
val_acc, val_loss, val_cfs_matrix, _ = m.eval_model(
model,
val_dataloader,
loss_function,
device,
output_type,
len(articles_val)
)
train_recall, train_precision, train_F1_score = result.classification_report(train_cfs_matrix)
]
train_settings = dict((k, settings[k]) for k in train_vars)
t0 = time()
best_epoch = 0
if info:
print(
'epoch\ttime\tinfo_loss\tD_loss\tG_loss\tmmd2\tthat\tpdf_sample\tpdf_real'
)
else:
print('epoch\ttime\tD_loss\tG_loss\tmmd2\tthat\tpdf_sample\tpdf_real')
for epoch in range(num_epochs):
if info:
info_loss_curr, D_loss_curr, G_loss_curr = model.train_epoch(
info, latent_C, latent_C_dim, epoch, samples['train'],
cond_samples_train, labels['train'], sess, Z, X, CG, CD, CS,
cond_sine, info_loss, D_loss, G_loss, info_solver, D_solver,
G_solver, **train_settings)
else:
D_loss_curr, G_loss_curr = model.train_epoch(
info, latent_C, latent_C_dim, epoch, samples['train'],
cond_samples_train, labels['train'], sess, Z, X, CG, CD, CS,
cond_sine, None, D_loss, G_loss, info_solver, D_solver, G_solver,
**train_settings)
# -- eval -- #
# visualise plots of generated samples, with/without labels
if epoch % vis_freq == 0:
if info:
best_model_mse = 30
best_model = ""
train_data = data.loc[train_idx].reset_index()
test_data = data.loc[test_idx].reset_index()
train_ds = TSDataset(train_data[XCOLS], train_data[YCOLS], window = HOURS)
test_ds = TSDataset(test_data[XCOLS], test_data[YCOLS], window = HOURS)
train_dataloader = get_loader(train_ds, batch_size=bs)
test_dataloader = get_loader(test_ds, batch_size=bs)
for epoch in range(n_epochs):
train_loss, val_loss = train_epoch(
model,
criterion,
optimizer,
train_dataloader,
test_dataloader,
device
)
print(
f"Split {split } Epoch {epoch}:\n\t\
Train loss: {train_loss:.2f}, Val loss: {val_loss:.2f}"
)
if val_loss < best_model_mse:
new_file = os.path.join(
args.save_dir,
timestamp + "_" + "best-{}-{:.3f}".format(epoch, val_loss))
best_model_mse = val_loss
best_model = deepcopy(model)
