def train(self):
lookahead = train_dataset.lookahead
val_lookahead = val_dataset.lookahead
t_val = torch.linspace(0, val_lookahead * dt, val_lookahead + 1)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=self.shuffle, drop_last=True)
val_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=self.shuffle, drop_last=True)
val_losses = []
train_losses = []
pre_val_loss = 1000000
assert(n_iterations == int(len(self.train_dataset) / self.batch_size))
logging.info("\nSTARTING TO TRAIN MODEL | EPOCHS : {} | lr: {} | batchsize: {} | lookahead: {}"
.format(self.epochs, self.lr, self.batch_size, lookahead))
logging.info(" | iterations: {} | #trainable parameters: {} \n"
.format(n_iterations, get_num_trainable_params(params)))
pre_train_time = time.time()
now = pre_train_time
for epoch in range(self.epochs):
for n, (batch_X0, batch_X) in enumerate(train_dataloader):
self.model.train()
self.optimizer.zero_grad()
batch_X0 = batch_X0.view(self.batch_size, self.data_dim)
X_pred = odeint(f, batch_X0, t).permute(1, 0, 2)
loss = torch.mean(torch.abs(X_pred - batch_X) ** 2)
loss.backward()
self.optimizer.step()
if self.train_on_one_batch:
break
train_losses.append(float(loss.detach()))
self.model.eval()
val_X0, val_X = next(iter(val_dataloader))
val_X0 = val_X0.view(self.batch_size, self.data_dim)
X_val_pred = odeint(self.model, val_X0, t_val).permute(1, 0, 2)
val_loss = torch.mean(torch.abs(X_val_pred - val_X) ** 2)
val_losses.append(float(val_loss))
# scheduler.step(val_loss)
if epoch % 10 == 0:
time_for_epochs = int(time.time() - now)
logging.info("EPOCH {} finished with training loss: {} | validation loss: {} | lr: {} | delta time: {}s"
.format(epoch, loss, val_loss, get_lr(self.optimizer), time_for_epochs))
save_models(self.model_dir, self.model)
save_optimizer(self.model_dir, self.optimizer)
# if val_loss > pre_val_loss and EPOCH % 30 == 0:
# logging.info("\n STOPPING TRAINING EARLY BECAUSE VAL.LOSS STOPPED IMPROVING!\n")
# break
now = time.time()
pre_val_loss = val_loss
post_train_time = time.time()
logging.info('\nTRAINING FINISHED after {} seconds'.format(int(post_train_time - pre_train_time)))
plt.plot(np.log(train_losses), label='train loss')
plt.plot(np.log(val_losses), label='validation loss')
plt.legend(), plt.savefig(figures_dir + '/losses.png'), plt.show()
def save_model(self,path = var_save_path, suffix = None):
if suffix is None:
suffix = self.model_name
save_list = [self.s_enc, self.z_enc, self.sz_dec, self.z_adv]
save_list[0].m_name = 's_enc'; save_list[1].m_name = 'z_enc';
save_list[2].m_name = 'sz_dec'; save_list[3].m_name = 'z_adv';
utils.save_models(save_list,path, mode='param', mode_param = suffix)
def main():
args = parse_arguments()
n_vocab = params.n_vocab
n_layer = params.n_layer
n_hidden = params.n_hidden
n_embed = params.n_embed
n_batch = args.n_batch
temperature = params.temperature
train_path = params.train_path
assert torch.cuda.is_available()
print("loading_data...")
# 训练时加载处理好的词典（如果有的话）
if os.path.exists("vocab.json"):
vocab = Vocabulary()
with open('vocab.json', 'r') as fp:
vocab.stoi = json.load(fp)
for key, value in vocab.stoi.items():
vocab.itos.append(key)
else:
vocab = build_vocab(train_path, n_vocab)
# save vocab
with open('vocab.json', 'w') as fp:
json.dump(vocab.stoi, fp)
train_X, train_y, train_K = load_data(train_path, vocab)
train_loader = get_data_loader(train_X, train_y, train_K, n_batch)
print("successfully loaded")
encoder = Encoder(n_vocab, n_embed, n_hidden, n_layer, vocab).cuda()
Kencoder = KnowledgeEncoder(n_vocab, n_embed, n_hidden, n_layer,
vocab).cuda()
manager = Manager(n_hidden, n_vocab, temperature).cuda()
decoder = Decoder(n_vocab, n_embed, n_hidden, n_layer, vocab).cuda()
if args.restore:
encoder = init_model(encoder, restore=params.encoder_restore)
Kencoder = init_model(Kencoder, restore=params.Kencoder_restore)
manager = init_model(manager, restore=params.manager_restore)
decoder = init_model(decoder, restore=params.decoder_restore)
# ToDo：目前的代码所有的embedding都是独立的，可以参考transformer源码使用直接赋值的方法共享参数：
#if emb_src_trg_weight_sharing:
# self.encoder.src_word_emb.weight = self.decoder.trg_word_emb.weight
model = [encoder, Kencoder, manager, decoder]
parameters = list(encoder.parameters()) + list(Kencoder.parameters()) + \
list(manager.parameters()) + list(decoder.parameters())
optimizer = optim.Adam(parameters, lr=args.lr)
# pre_train knowledge manager
print("start pre-training")
pre_train(model, optimizer, train_loader, args)
print("start training")
train(model, optimizer, train_loader, args)
# save final model
save_models(model, params.all_restore)
def switch_evaluation_to_training(engine):
if needs_save:
save_logs('val', k, n_splits, evaluator, trainer.state.epoch, trainer.state.iteration,
config, output_dir_path)
if trainer.state.epoch % 100 == 0:
save_models(model, optimizer, k, n_splits, trainer.state.epoch, trainer.state.iteration,
config, output_dir_path)
def train(self, d_model_A, d_model_B, g_model_AtoB, g_model_BtoA,
c_model_AtoB, c_model_BtoA, dataset):
''' tarining step for cyclegan models'''
# define properties of the training run
n_epochs, n_batch, = 50, 1
# determine the output square shape of the discriminator
n_patch = d_model_A.output_shape[1]
# unpack dataset
trainA, trainB = dataset
# prepare image pool for fakes
poolA, poolB = list(), list()
# calculate the number of batches per training epoch
bat_per_epo = int(len(trainA) / n_batch)
# calculate the number of training iterations
n_steps = bat_per_epo * n_epochs
# manually enumerate epochs
for i in range(n_steps):
# select a batch of real samples
X_realA, y_realA = generate_real_samples(trainA, n_batch, n_patch)
X_realB, y_realB = generate_real_samples(trainB, n_batch, n_patch)
# generate a batch of fake samples
X_fakeA, y_fakeA = generate_fake_samples(g_model_BtoA, X_realB,
n_patch)
X_fakeB, y_fakeB = generate_fake_samples(g_model_AtoB, X_realA,
n_patch)
# update fakes from pool
X_fakeA = update_image_pool(poolA, X_fakeA)
X_fakeB = update_image_pool(poolB, X_fakeB)
# update generator B->A via adversarial and cycle los
g_loss2, _, _, _, _ = c_model_BtoA.train_on_batch(
[X_realB, X_realA], [y_realA, X_realA, X_realB, X_realA])
# update discriminator for A -> [real/fake]
dA_loss1 = d_model_A.train_on_batch(X_realA, y_realA)
dA_loss2 = d_model_A.train_on_batch(X_fakeA, y_fakeA)
# update generator A->B via adversarial and cycle loss
g_loss1, _, _, _, _ = c_model_AtoB.train_on_batch(
[X_realA, X_realB], [y_realB, X_realB, X_realA, X_realB])
# update discriminator for B -> [real/fake]
dB_loss1 = d_model_B.train_on_batch(X_realB, y_realB)
dB_loss2 = d_model_B.train_on_batch(X_fakeB, y_fakeB)
# summarize performance
print('>%d, dA[%.3f,%.3f] dB[%.3f,%.3f] g[%.3f,%.3f]' %
(i + 1, dA_loss1, dA_loss2, dB_loss1, dB_loss2, g_loss1,
g_loss2))
# evaluate the model performance every so often
if (i + 1) % (bat_per_epo * 1) == 0:
# plot A->B translation
summarize_performance(i, g_model_AtoB, trainA, 'AtoB')
# plot B->A translation
summarize_performance(i, g_model_BtoA, trainB, 'BtoA')
if (i + 1) % (bat_per_epo * 5) == 0:
# save the models
save_models(i, g_model_AtoB, g_model_BtoA)
def main():
args = parse_arguments()
n_vocab = params.n_vocab
n_layer = params.n_layer
n_hidden = params.n_hidden
n_embed = params.n_embed
n_batch = args.n_batch
temperature = params.temperature
train_path = params.train_path
assert torch.cuda.is_available()
print("loading_data...")
vocab = build_vocab(train_path, n_vocab)
# save vocab
with open('vocab.json', 'w') as fp:
json.dump(vocab.stoi, fp)
train_X, train_y, train_K = load_data(train_path, vocab)
train_loader = get_data_loader(train_X, train_y, train_K, n_batch)
print("successfully loaded")
encoder = Encoder(n_vocab, n_embed, n_hidden, n_layer, vocab).cuda()
Kencoder = KnowledgeEncoder(n_vocab, n_embed, n_hidden, n_layer,
vocab).cuda()
manager = Manager(n_hidden, n_vocab, temperature).cuda()
decoder = Decoder(n_vocab, n_embed, n_hidden, n_layer, vocab).cuda()
if args.restore:
encoder = init_model(encoder, restore=params.encoder_restore)
Kencoder = init_model(Kencoder, restore=params.Kencoder_restore)
manager = init_model(manager, restore=params.manager_restore)
decoder = init_model(decoder, restore=params.decoder_restore)
model = [encoder, Kencoder, manager, decoder]
parameters = list(encoder.parameters()) + list(Kencoder.parameters()) + \
list(manager.parameters()) + list(decoder.parameters())
optimizer = optim.Adam(parameters, lr=args.lr)
# pre_train knowledge manager
print("start pre-training")
pre_train(model, optimizer, train_loader, args)
print("start training")
train(model, optimizer, train_loader, args)
# save final model
save_models(model, params.all_restore)
def main():
env = airsim_env.windENV()
memory = ReplayBuffer()
q, q_target = QNet(), QNet()
q_target.load_state_dict(q.state_dict())
mu, mu_target = MuNet(), MuNet()
mu_target.load_state_dict(mu.state_dict())
score = 0.0
print_interval = 20
mu_optimizer = optim.Adam(mu.parameters(), lr=lr_mu)
q_optimizer = optim.Adam(q.parameters(), lr=lr_q)
for n_epi in range(10000):
s = env.reset()
done = False
while not done:
a = mu(torch.from_numpy(s).float())
a = a.detach().numpy()
a = (a + np.random.normal(0, 0.1, size=4)).clip(-0.6, 0.4)
s_prime, r, done, info = env.step(a)
memory.put((s, a, r / 100.0, s_prime, done))
score += r
s = s_prime
if memory.size() > 1000:
train(mu, mu_target, q, q_target, memory, q_optimizer,
mu_optimizer)
soft_update(mu, mu_target)
soft_update(q, q_target)
if n_epi % print_interval == 0 and n_epi != 0:
print("# of episode :{}, avg score : {:.1f}".format(
n_epi, score / print_interval))
score = 0.0
if n_epi % 1000 == 0 and n_epi != 0:
utils.save_models(mu, q, n_epi)
def pre_train(model, optimizer, train_loader, args):
encoder, Kencoder, manager, decoder = [*model]
encoder.train(), Kencoder.train(), manager.train(), decoder.train()
parameters = list(encoder.parameters()) + list(Kencoder.parameters()) + \
list(manager.parameters()) + list(decoder.parameters())
NLLLoss = nn.NLLLoss(reduction='mean', ignore_index=params.PAD)
for epoch in range(args.pre_epoch):
b_loss = 0
for step, (src_X, src_y, src_K, _) in enumerate(train_loader):
src_X = src_X.cuda()
src_y = src_y.cuda()
src_K = src_K.cuda()
mask = (src_X != 0).long()
optimizer.zero_grad()
_, x = encoder(src_X, mask)
n_batch = src_y.size(0)
n_vocab = params.n_vocab
CLS_tokens = torch.LongTensor([params.CLS] * n_batch).unsqueeze(1).cuda()
SEP_tokens = torch.LongTensor([params.SEP] * n_batch).unsqueeze(1).cuda()
src_y_ = torch.cat((CLS_tokens, src_y[:, 1:], SEP_tokens), dim=-1)
y = Kencoder(src_y_)
K = Kencoder(src_K)
seq_len = src_y.size(1) - 1
_, _, _, k_logits = manager(x, y, K) # k_logits: [n_batch, n_vocab]
k_logits = k_logits.repeat(seq_len, 1, 1).transpose(0, 1).contiguous().view(-1, n_vocab)
bow_loss = NLLLoss(k_logits, src_y[:, 1:].contiguous().view(-1))
bow_loss.backward()
clip_grad_norm_(parameters, args.grad_clip)
optimizer.step()
b_loss += bow_loss.item()
if (step + 1) % 10 == 0:
b_loss /= 10
print("Epoch [%.1d/%.1d] Step [%.4d/%.4d]: bow_loss=%.4f" % (epoch + 1, args.pre_epoch,
step + 1, len(train_loader),
b_loss))
b_loss = 0
# save models
save_models(model, params.all_restore)
def pre_train(model, optimizer, train_loader, args):
encoder, Kencoder, manager, decoder = [*model]
encoder.train(), Kencoder.train(), manager.train(), decoder.train()
parameters = list(encoder.parameters()) + list(Kencoder.parameters()) + \
list(manager.parameters())
NLLLoss = nn.NLLLoss(reduction='mean', ignore_index=params.PAD)
for epoch in range(args.pre_epoch):
epoch_start_time =time.time()
b_loss = 0
print('Start preTraining epoch %d ...'%(epoch))
for step, (src_X, src_y, src_K, _) in enumerate(train_loader):
# if step>300:
# break
src_X = src_X.cuda()
src_y = src_y.cuda()
src_K = src_K.cuda()
optimizer.zero_grad()
_, _, x = encoder(src_X)
y = Kencoder(src_y)
K = Kencoder(src_K)
n_vocab = params.n_vocab
seq_len = src_y.size(1) - 1
_, _, _, k_logits = manager(x, y, K) # k_logits: [n_batch, n_vocab]
k_logits = k_logits.repeat(seq_len, 1, 1).transpose(0, 1).contiguous().view(-1, n_vocab)
bow_loss = NLLLoss(k_logits, src_y[:, 1:].contiguous().view(-1))
bow_loss.backward()
clip_grad_norm_(parameters, args.grad_clip)
optimizer.step()
b_loss += bow_loss.item()
if (step + 1) % 10 == 0:
b_loss /= 10
print("Epoch [%.1d/%.1d] Step [%.4d/%.4d]: bow_loss=%.4f, epoch_time:%.2f s" % (epoch + 1, args.pre_epoch,
step + 1, len(train_loader),
b_loss,time.time()-epoch_start_time))
b_loss = 0
# save models
save_models(model, params.all_restore,0-epoch-1,b_loss)
def main(args):
# hyperparameters
batch_size = args.batch_size
num_workers = 1
# Image Preprocessing
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
])
# load COCOs dataset
IMAGES_PATH = 'data/train2014'
CAPTION_FILE_PATH = 'data/annotations/captions_train2014.json'
vocab = load_vocab()
train_loader = get_coco_data_loader(path=IMAGES_PATH,
json=CAPTION_FILE_PATH,
vocab=vocab,
transform=transform,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
IMAGES_PATH = 'data/val2014'
CAPTION_FILE_PATH = 'data/annotations/captions_val2014.json'
val_loader = get_coco_data_loader(path=IMAGES_PATH,
json=CAPTION_FILE_PATH,
vocab=vocab,
transform=transform,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
losses_val = []
losses_train = []
# Build the models
ngpu = 1
initial_step = initial_epoch = 0
embed_size = args.embed_size
num_hiddens = args.num_hidden
learning_rate = 5e-4
num_epochs = 2
log_step = args.log_step
save_step = 500
checkpoint_dir = args.checkpoint_dir
encoder = CNNEncoder()
decoder = DecoderRNN(embed_size, num_hiddens, len(vocab))
# Loss
criterion = nn.CrossEntropyLoss()
if args.checkpoint_file:
encoder_state_dict, decoder_state_dict, optimizer, *meta = utils.load_models(
args.checkpoint_file, args.sample)
initial_step, initial_epoch, losses_train, losses_val = meta
encoder.load_state_dict(encoder_state_dict)
decoder.load_state_dict(decoder_state_dict)
else:
params = list(decoder.parameters()) + list(
encoder.batchnorm.parameters())
optimizer = torch.optim.Adam(params, lr=learning_rate)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
if args.sample:
return utils.sample(encoder, decoder, vocab, val_loader)
# Train the Models
total_step = len(train_loader)
try:
for epoch in range(initial_epoch, num_epochs):
for step, (images, captions,
lengths) in enumerate(train_loader, start=initial_step):
# Set mini-batch dataset
images = utils.to_var(images, volatile=True)
captions = utils.to_var(captions)
targets = pack_padded_sequence(captions,
lengths,
batch_first=True)[0]
# Forward, Backward and Optimize
decoder.zero_grad()
encoder.zero_grad()
if ngpu > 1:
# run on multiple GPU
features = nn.parallel.data_parallel(
encoder, images, range(ngpu))
outputs, alphas = nn.parallel.data_parallel(
decoder, features, range(ngpu))
else:
# run on single GPU
features = encoder(images)
outputs, alphas = decoder(features, captions, lengths)
train_loss = criterion(outputs, targets.cpu())
train_loss += ((1. - alphas.sum(dim=1))**2).mean()
losses_train.append(train_loss.data)
train_loss.backward()
optimizer.step()
print('Epoch: {} - Step: {} - Train Loss: {}'.format(
epoch, step, losses_train[-1]))
# Run validation set and predict
if step % log_step == 404:
encoder.batchnorm.eval()
# run validation set
batch_loss_val = []
for val_step, (images, captions,
lengths) in enumerate(val_loader):
images = utils.to_var(images, volatile=True)
captions = utils.to_var(captions, volatile=True)
targets = pack_padded_sequence(captions,
lengths,
batch_first=True)[0]
features = encoder(images)
outputs, alphas = decoder(features, captions, lengths)
val_loss = criterion(outputs, targets.cpu())
val_loss += ((1. - alphas.sum(dim=1))**2).mean()
batch_loss_val.append(val_loss.data)
if val_step % 50 == 0:
print('Epoch: {} - Step: {} - Mini Eval Loss: {}'.
format(epoch, val_step, val_loss))
sampled_ids = decoder.sample(features)
sampled_ids = sampled_ids.cpu().data.numpy()[0]
sentence = utils.convert_back_to_text(
sampled_ids, vocab)
print('Sample:', sentence)
true_ids = captions.cpu().data.numpy()[0]
sentence = utils.convert_back_to_text(
true_ids, vocab)
print('Target:', sentence)
losses_val.append(np.mean(batch_loss_val))
# predict
print('Epoch: {} - Step: {} - Eval Loss: {}'.format(
epoch, step, losses_val[-1]))
encoder.batchnorm.train()
# Save the models
if (step + 1) % save_step == 0:
utils.save_models(encoder, decoder, optimizer, step, epoch,
losses_train, losses_val, checkpoint_dir)
utils.dump_losses(
losses_train, losses_val,
os.path.join(checkpoint_dir, 'losses.pkl'))
except KeyboardInterrupt:
pass
finally:
# Do final save
utils.save_models(encoder, decoder, optimizer, step, epoch,
losses_train, losses_val, checkpoint_dir)
utils.dump_losses(losses_train, losses_val,
os.path.join(checkpoint_dir, 'losses.pkl'))
def train(model, optimizer, train_loader, args):
encoder, Kencoder, manager, decoder = [*model]
encoder.train(), Kencoder.train(), manager.train(), decoder.train()
parameters = list(encoder.parameters()) + list(Kencoder.parameters()) + \
list(manager.parameters()) + list(decoder.parameters())
NLLLoss = nn.NLLLoss(reduction='mean', ignore_index=params.PAD)
KLDLoss = nn.KLDivLoss(reduction='batchmean')
for epoch in range(args.n_epoch):
b_loss = 0
k_loss = 0
n_loss = 0
t_loss = 0
for step, (src_X, src_y, src_K, tgt_y) in enumerate(train_loader):
src_X = src_X.cuda()
src_y = src_y.cuda()
src_K = src_K.cuda()
tgt_y = tgt_y.cuda()
optimizer.zero_grad()
encoder_outputs, hidden, x = encoder(src_X)
encoder_mask = (src_X == 0)[:, :encoder_outputs.size(0)].unsqueeze(1).byte()
y = Kencoder(src_y)
K = Kencoder(src_K)
prior, posterior, k_i, k_logits = manager(x, y, K)
kldiv_loss = KLDLoss(prior, posterior.detach())
n_vocab = params.n_vocab
seq_len = src_y.size(1) - 1
k_logits = k_logits.repeat(seq_len, 1, 1).transpose(0, 1).contiguous().view(-1, n_vocab)
bow_loss = NLLLoss(k_logits, src_y[:, 1:].contiguous().view(-1))
n_batch = src_X.size(0)
max_len = tgt_y.size(1)
outputs = torch.zeros(max_len, n_batch, n_vocab).cuda()
hidden = hidden[params.n_layer:]
output = torch.LongTensor([params.SOS] * n_batch).cuda() # [n_batch]
for t in range(max_len):
output, hidden, attn_weights = decoder(output, k_i, hidden, encoder_outputs, encoder_mask)
outputs[t] = output
is_teacher = random.random() < args.tfr # teacher forcing ratio
top1 = output.data.max(1)[1]
output = tgt_y[:, t] if is_teacher else top1
outputs = outputs.transpose(0, 1).contiguous()
nll_loss = NLLLoss(outputs.view(-1, n_vocab),
tgt_y.contiguous().view(-1))
loss = kldiv_loss + nll_loss + bow_loss
loss.backward()
clip_grad_norm_(parameters, args.grad_clip)
optimizer.step()
b_loss += bow_loss.item()
k_loss += kldiv_loss.item()
n_loss += nll_loss.item()
t_loss += loss.item()
if (step + 1) % 50 == 0:
k_loss /= 50
n_loss /= 50
b_loss /= 50
t_loss /= 50
print("Epoch [%.2d/%.2d] Step [%.4d/%.4d]: total_loss=%.4f kldiv_loss=%.4f bow_loss=%.4f nll_loss=%.4f"
% (epoch + 1, args.n_epoch,
step + 1, len(train_loader),
t_loss, k_loss, b_loss, n_loss))
k_loss = 0
n_loss = 0
b_loss = 0
t_loss = 0
# save models
save_models(model, params.all_restore)
def main():
"""Run NFQ."""
# Setup hyperparameters
parser = configargparse.ArgParser()
parser.add("-c", "--config", required=True, is_config_file=True)
parser.add("--EPOCH", type=int)
parser.add("--TRAIN_ENV_MAX_STEPS", type=int)
parser.add("--EVAL_ENV_MAX_STEPS", type=int)
parser.add("--DISCOUNT", type=float)
parser.add("--INIT_EXPERIENCE", type=int)
parser.add("--INCREMENT_EXPERIENCE", action="store_true")
parser.add("--HINT_TO_GOAL", action="store_true")
parser.add("--RANDOM_SEED", type=int)
parser.add("--TRAIN_RENDER", action="store_true")
parser.add("--EVAL_RENDER", action="store_true")
parser.add("--SAVE_PATH", type=str, default="")
parser.add("--LOAD_PATH", type=str, default="")
parser.add("--USE_TENSORBOARD", action="store_true")
parser.add("--USE_WANDB", action="store_true")
CONFIG = parser.parse_args()
if not hasattr(CONFIG, "INCREMENT_EXPERIENCE"):
CONFIG.INCREMENT_EXPERIENCE = False
if not hasattr(CONFIG, "HINT_TO_GOAL"):
CONFIG.HINT_TO_GOAL = False
if not hasattr(CONFIG, "TRAIN_RENDER"):
CONFIG.TRAIN_RENDER = False
if not hasattr(CONFIG, "EVAL_RENDER"):
CONFIG.EVAL_RENDER = False
if not hasattr(CONFIG, "USE_TENSORBOARD"):
CONFIG.USE_TENSORBOARD = False
if not hasattr(CONFIG, "USE_WANDB"):
CONFIG.USE_WANDB = False
print()
print(
"+--------------------------------+--------------------------------+")
print(
"| Hyperparameters | Value |")
print(
"+--------------------------------+--------------------------------+")
for arg in vars(CONFIG):
print("| {:30} | {:<30} |".format(
arg,
getattr(CONFIG, arg) if getattr(CONFIG, arg) is not None else ""))
print(
"+--------------------------------+--------------------------------+")
print()
# Log to File, Console, TensorBoard, W&B
logger = get_logger()
if CONFIG.USE_TENSORBOARD:
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(log_dir="tensorboard_logs")
if CONFIG.USE_WANDB:
import wandb
wandb.init(project="implementations-nfq", config=CONFIG)
# Setup environment
train_env = CartPoleRegulatorEnv(mode="train")
eval_env = CartPoleRegulatorEnv(mode="eval")
# Fix random seeds
if CONFIG.RANDOM_SEED is not None:
make_reproducible(CONFIG.RANDOM_SEED, use_numpy=True, use_torch=True)
train_env.seed(CONFIG.RANDOM_SEED)
eval_env.seed(CONFIG.RANDOM_SEED)
else:
logger.warning(
"Running without a random seed: this run is NOT reproducible.")
# Setup agent
nfq_net = NFQNetwork()
optimizer = optim.Rprop(nfq_net.parameters())
nfq_agent = NFQAgent(nfq_net, optimizer)
# Load trained agent
if CONFIG.LOAD_PATH:
load_models(CONFIG.LOAD_PATH, nfq_net=nfq_net, optimizer=optimizer)
# NFQ Main loop
# A set of transition samples denoted as D
all_rollouts = []
total_cost = 0
if CONFIG.INIT_EXPERIENCE:
for _ in range(CONFIG.INIT_EXPERIENCE):
rollout, episode_cost = train_env.generate_rollout(
None, render=CONFIG.TRAIN_RENDER)
all_rollouts.extend(rollout)
total_cost += episode_cost
for epoch in range(CONFIG.EPOCH + 1):
# Variant 1: Incermentally add transitions (Section 3.4)
# TODO(seungjaeryanlee): Done before or after training?
if CONFIG.INCREMENT_EXPERIENCE:
new_rollout, episode_cost = train_env.generate_rollout(
nfq_agent.get_best_action, render=CONFIG.TRAIN_RENDER)
all_rollouts.extend(new_rollout)
total_cost += episode_cost
state_action_b, target_q_values = nfq_agent.generate_pattern_set(
all_rollouts)
# Variant 2: Clamp function to zero in goal region
# TODO(seungjaeryanlee): Since this is a regulator setting, should it
# not be clamped to zero?
if CONFIG.HINT_TO_GOAL:
goal_state_action_b, goal_target_q_values = train_env.get_goal_pattern_set(
)
goal_state_action_b = torch.FloatTensor(goal_state_action_b)
goal_target_q_values = torch.FloatTensor(goal_target_q_values)
state_action_b = torch.cat([state_action_b, goal_state_action_b],
dim=0)
target_q_values = torch.cat(
[target_q_values, goal_target_q_values], dim=0)
loss = nfq_agent.train((state_action_b, target_q_values))
# TODO(seungjaeryanlee): Evaluation should be done with 3000 episodes
eval_episode_length, eval_success, eval_episode_cost = nfq_agent.evaluate(
eval_env, CONFIG.EVAL_RENDER)
if CONFIG.INCREMENT_EXPERIENCE:
logger.info(
"Epoch {:4d} | Train {:3d} / {:4.2f} | Eval {:4d} / {:5.2f} | Train Loss {:.4f}"
.format( # noqa: B950
epoch,
len(new_rollout),
episode_cost,
eval_episode_length,
eval_episode_cost,
loss,
))
if CONFIG.USE_TENSORBOARD:
writer.add_scalar("train/episode_length", len(new_rollout),
epoch)
writer.add_scalar("train/episode_cost", episode_cost, epoch)
writer.add_scalar("train/loss", loss, epoch)
writer.add_scalar("eval/episode_length", eval_episode_length,
epoch)
writer.add_scalar("eval/episode_cost", eval_episode_cost,
epoch)
if CONFIG.USE_WANDB:
wandb.log({"Train Episode Length": len(new_rollout)},
step=epoch)
wandb.log({"Train Episode Cost": episode_cost}, step=epoch)
wandb.log({"Train Loss": loss}, step=epoch)
wandb.log({"Evaluation Episode Length": eval_episode_length},
step=epoch)
wandb.log({"Evaluation Episode Cost": eval_episode_cost},
step=epoch)
else:
logger.info(
"Epoch {:4d} | Eval {:4d} / {:5.2f} | Train Loss {:.4f}".
format(epoch, eval_episode_length, eval_episode_cost, loss))
if CONFIG.USE_TENSORBOARD:
writer.add_scalar("train/loss", loss, epoch)
writer.add_scalar("eval/episode_length", eval_episode_length,
epoch)
writer.add_scalar("eval/episode_cost", eval_episode_cost,
epoch)
if CONFIG.USE_WANDB:
wandb.log({"Train Loss": loss}, step=epoch)
wandb.log({"Evaluation Episode Length": eval_episode_length},
step=epoch)
wandb.log({"Evaluation Episode Cost": eval_episode_cost},
step=epoch)
if eval_success:
logger.info(
"Epoch {:4d} | Total Cycles {:6d} | Total Cost {:4.2f}".format(
epoch, len(all_rollouts), total_cost))
if CONFIG.USE_TENSORBOARD:
writer.add_scalar("summary/total_cycles", len(all_rollouts),
epoch)
writer.add_scalar("summary/total_cost", total_cost, epoch)
if CONFIG.USE_WANDB:
wandb.log({"Total Cycles": len(all_rollouts)}, step=epoch)
wandb.log({"Total Cost": total_cost}, step=epoch)
break
# Save trained agent
if CONFIG.SAVE_PATH:
save_models(CONFIG.SAVE_PATH, nfq_net=nfq_net, optimizer=optimizer)
train_env.close()
eval_env.close()
def get_bssevals(dr, arguments):
loader1, loader2, loader_mix = ut.preprocess_timit_files(arguments, dr=dr)
exp_info = '_'.join([
arguments.tr_method, arguments.test_method, arguments.data,
arguments.dataname, arguments.input_type, arguments.optimizer,
'feat_match',
str(arguments.feat_match), 'K',
str(arguments.K), 'Kdisc',
str(arguments.Kdisc), 'L1',
str(arguments.L1), 'smooth_estimate',
str(arguments.smooth_source_estimates), 'nfft',
str(arguments.n_fft)
])
arguments.exp_info = exp_info
ngpu = 1
L1 = arguments.L1
L2 = arguments.L2
K = arguments.K
Kdisc = arguments.Kdisc
smooth_output = arguments.smooth_output
tr_method = arguments.tr_method
loss = 'Poisson'
alpha_range = arguments.alpha_range
if tr_method == 'VAE':
generator1 = VAE(ngpu, K=K, L1=L1, L2=L2, arguments=arguments)
generator2 = VAE(ngpu, K=K, L1=L1, L2=L2, arguments=arguments)
else:
generator1 = netG(ngpu, K=K, L1=L1, L2=L2, arguments=arguments)
generator2 = netG(ngpu, K=K, L1=L1, L2=L2, arguments=arguments)
discriminator1 = netD(ngpu, K=Kdisc, L=L2, arguments=arguments)
discriminator2 = netD(ngpu, K=Kdisc, L=L2, arguments=arguments)
if arguments.cuda:
generator1.cuda()
discriminator1.cuda()
generator2.cuda()
discriminator2.cuda()
# Train the generative models for the sources
if arguments.load_models:
modelfldr = 'model_parameters'
generator1.load_state_dict(
torch.load(
os.path.join(modelfldr, 'generator0_' + exp_info + '.trc')))
generator2.load_state_dict(
torch.load(
os.path.join(modelfldr, 'generator1_' + exp_info + '.trc')))
discriminator1.load_state_dict(
torch.load(
os.path.join(modelfldr,
'discriminator0_' + exp_info + '.trc')))
discriminator2.load_state_dict(
torch.load(
os.path.join(modelfldr,
'discriminator1_' + exp_info + '.trc')))
else:
EP = arguments.EP_train
if tr_method == 'adversarial':
criterion = nn.BCELoss()
adversarial_trainer(loader_mix=loader_mix,
train_loader=loader1,
generator=generator1,
discriminator=discriminator1,
EP=EP,
arguments=arguments,
criterion=criterion,
conditional_gen=False,
source_num=1)
adversarial_trainer(loader_mix=loader_mix,
train_loader=loader2,
generator=generator2,
discriminator=discriminator2,
EP=EP,
arguments=arguments,
criterion=criterion,
conditional_gen=False,
source_num=2)
elif tr_method == 'adversarial_wasserstein':
adversarial_wasserstein_trainer(loader_mix=loader_mix,
train_loader=loader1,
generator=generator1,
discriminator=discriminator1,
EP=EP,
arguments=arguments,
conditional_gen=False,
source_num=1)
adversarial_wasserstein_trainer(loader_mix=loader_mix,
train_loader=loader2,
generator=generator2,
discriminator=discriminator2,
EP=EP,
arguments=arguments,
conditional_gen=False,
source_num=2)
elif tr_method == 'VAE':
def loss_function(recon_x, x, mu, logvar):
eps = 1e-20
criterion = lambda lam, tar: torch.mean(-tar * torch.log(
lam + eps) + lam)
BCE = criterion(recon_x, x)
# see Appendix B from VAE paper:
# Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014
# https://arxiv.org/abs/1312.6114
# 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
# Normalise by same number of elements as in reconstruction
KLD /= x.size(0) * arguments.L2
return BCE + KLD
VAE_trainer(loader_mix=loader_mix,
train_loader=loader1,
generator=generator1,
EP=EP,
arguments=arguments,
criterion=loss_function,
conditional_gen=False)
VAE_trainer(loader_mix=loader_mix,
train_loader=loader2,
generator=generator2,
EP=EP,
arguments=arguments,
criterion=loss_function,
conditional_gen=False)
elif tr_method == 'ML':
if loss == 'Euclidean':
criterion = nn.MSELoss()
elif loss == 'Poisson':
eps = 1e-20
criterion = lambda lam, tar: torch.mean(-tar * torch.log(
lam + eps) + lam)
generative_trainer(loader_mix=loader_mix,
train_loader=loader1,
generator=generator1,
EP=EP,
arguments=arguments,
criterion=criterion,
conditional_gen=False)
generative_trainer(loader_mix=loader_mix,
train_loader=loader2,
generator=generator2,
EP=EP,
arguments=arguments,
criterion=criterion,
conditional_gen=False)
# save models
savepath = os.path.join(os.getcwd(), 'model_parameters')
if not os.path.exists(savepath):
os.mkdir(savepath)
ut.save_models([generator1, generator2],
[discriminator1, discriminator2], exp_info, savepath,
arguments)
# Separate out the sources
bss_evals = []
for alpha in alpha_range:
print('The current tradeoff parameter is {}'.format(alpha))
bss_eval = ML_separate_audio_sources(
generators=[generator1, generator2],
discriminators=[discriminator1, discriminator2],
loader_mix=loader_mix,
EP=arguments.EP_test,
arguments=arguments,
conditional=False,
tr_method=tr_method,
loss=loss,
alpha=float(alpha),
exp_info=exp_info)
bss_evals.append(bss_eval)
return bss_evals
def train_model(encoder_optimizer, decoder_optimizer, encoder, decoder, loss_fun, m_type, dataloader, en_lang, vi_lang, save_path, encoder_save, decoder_save, num_epochs=60, val_every = 1, train_bleu_every = 10,clip = 0.1, rm = 0.8, enc_scheduler = None, dec_scheduler = None, enc_dec_fn = encode_decode, val_fn = validation_new):
'''
'''
best_score = 0
best_bleu = 0
loss_hist = {'train': [], 'validate': []}
bleu_hist = {'train': [], 'validate': []}
best_encoder_wts = None
best_decoder_wts = None
phases = ['train','validate']
for epoch in range(num_epochs):
print('Epoch: [{}/{}]'.format(epoch, num_epochs));
log.info('Epoch: [{}/{}]'.format(epoch, num_epochs))
for ex, phase in enumerate(phases):
start = time.time()
total = 0
top1_correct = 0
running_loss = 0
running_total = 0
if phase == 'train':
encoder.train()
decoder.train()
else:
encoder.eval()
decoder.eval()
for i, data in enumerate(dataloader[phase]):
encoder_optimizer.zero_grad()
decoder_optimizer.zero_grad()
encoder_i = data[0].to(device)
decoder_i = data[1].to(device)
src_len = data[2].to(device)
tar_len = data[3].to(device)
if phase == 'validate':
out = enc_dec_fn(encoder, decoder, encoder_i, decoder_i, src_len, tar_len, rand_num=rm, val = True)
else:
out = enc_dec_fn(encoder, decoder, encoder_i, decoder_i, src_len, tar_len, rand_num=rm, val = False)
N = decoder_i.size(0)
loss = loss_fun(out.float(), decoder_i.long())
running_loss += loss.item() * N
total += N
if phase == 'train':
loss.backward()
torch.nn.utils.clip_grad_norm_(encoder.parameters(), clip)
torch.nn.utils.clip_grad_norm_(decoder.parameters(), clip)
encoder_optimizer.step()
decoder_optimizer.step()
epoch_loss = running_loss / total
loss_hist[phase].append(epoch_loss)
print("epoch {} {} loss = {}, time = {}".format(epoch, phase, epoch_loss, time.time() - start))
log.info("epoch {} {} loss = {}, time = {}".format(epoch, phase, epoch_loss, time.time() - start))
if (enc_scheduler is not None) and (dec_scheduler is not None):
enc_scheduler.step(loss_hist['train'][-1])
dec_scheduler.step(loss_hist['train'][-1])
if epoch%val_every == 0:
val_bleu_score, _, _ , _ = val_fn(encoder, decoder, dataloader['validate'], en_lang, vi_lang, m_type, verbose=False, replace_unk=True)
bleu_hist['validate'].append(val_bleu_score)
print("validation BLEU = {}".format(val_bleu_score))
log.info("validation BLEU = {}".format(val_bleu_score))
if val_bleu_score > best_bleu:
best_bleu = val_bleu_score
best_encoder_wts = encoder.state_dict()
best_decoder_wts = decoder.state_dict()
# save best model
utils.save_models(best_encoder_wts, save_path, encoder_save)
utils.save_models(best_decoder_wts, save_path, decoder_save)
print('='*50)
encoder.load_state_dict(best_encoder_wts)
decoder.load_state_dict(best_decoder_wts)
print("Training completed. Best BLEU is {}".format(best_bleu))
return encoder, decoder, loss_hist, bleu_hist
def nested_cv(params):
# The number of validation split
cv_num = params['cv_num']
# The number of test split
cv_num_test = params['cv_num_test']
# Data and labels
data, hc_ad, site_id_, split_ref_, ref = prepare_data(params['source_dir'])
# Numpy array for ML models' performance indices
measurements_transfer = np.zeros(shape=[cv_num_test, 5])
measurements_classifier = np.zeros(shape=[cv_num_test, 5])
measurements_svm = np.zeros(shape=[cv_num_test, 5])
measurements_rf = np.zeros(shape=[cv_num_test, 5])
# List of accuracy for final output
accs_transfer = []
accs_classifier = []
accs_svm = []
accs_rf = []
skf_ = StratifiedKFold(n_splits=cv_num_test, random_state=0, shuffle=True)
# Split keeping the ratio of acquisition sites and ASD/TC simultaneously
for cv_iteration_, (train_index_, test_index) in enumerate(skf_.split(data, split_ref_)):
# List of standard scalers for each validation split
scalers = []
# List of trained models
trained_transfer = []
trained_classifier = []
epochs = []
# Split data
x_train_, x_test = data[train_index_], data[test_index]
labels_train_, labels_test = hc_ad[train_index_], hc_ad[test_index]
split_ref = split_ref_[train_index_]
skf = StratifiedKFold(n_splits=cv_num, random_state=0, shuffle=True)
# Split keeping the ratio of acquisition sites and ASD/TC simultaneously
for cv_iteration, (train_index, valid_index) in enumerate(skf.split(x_train_, split_ref)):
epochs_cv = []
# Split data
x_train, x_valid = x_train_[train_index], x_train_[valid_index]
labels_train, labels_valid = labels_train_[train_index], labels_train_[valid_index]
# Standardize input
scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)
x_valid = scaler.transform(x_valid)
scalers.append(scaler)
# Train EIIC model
model_trained_transfer = train_transfer(x_train, labels_train, x_valid, labels_valid, params['nn_params'])
trained_transfer.append(model_trained_transfer)
# Train simple MLP without contrastive learning
model_trained_classifier = train_classifier(x_train, labels_train, x_valid, labels_valid, params['nn_params'])
trained_classifier.append(model_trained_classifier)
# Calculate confusion matrix from trained model
matrix = matrix_from_models(trained_transfer, scalers, x_test, labels_test, params['nn_params']['num_classes'], params['nn_params']['device'])
# Calculate performance indices from confusion matrix
acc, recall, specificity, ppv, npv = model_measurements(matrix)
accs_transfer.append(acc)
measurements_transfer[cv_iteration_,:] = [acc, recall, specificity, ppv, npv]
matrix = matrix_from_models_cl(trained_classifier, scalers, x_test, labels_test, params['nn_params']['num_classes'], params['nn_params']['device'])
acc, recall, specificity, ppv, npv = model_measurements(matrix)
accs_classifier.append(acc)
measurements_classifier[cv_iteration_,:] = [acc, recall, specificity, ppv, npv]
# PCA dimensionality reduction for SVM and RF
pca = PCA(n_components = params['nn_params']['prefinal_num'])
# Get the transformation of PCA from data excepting test data
pca.fit(x_train_)
# Transform data
x_train_sc = pca.transform(x_train_)
x_test_sc = pca.transform(x_test)
# Train SVM and RF
svm = train_svm(x_train_sc, labels_train_, skf.split(x_train_, split_ref), params['tuning_params_svm'])
rf = train_rf(x_train_sc, labels_train_, skf.split(x_train_, split_ref), params['tuning_params_rf'])
# Calculate confusion matrix and performance indices
matrix = prediction_matrix(x_test_sc, labels_test, svm, params['nn_params']['num_classes'])
acc, recall, specificity, ppv, npv = model_measurements(matrix)
accs_svm.append(acc)
measurements_svm[cv_iteration_,:] = [acc, recall, specificity, ppv, npv]
matrix = prediction_matrix(x_test_sc, labels_test, rf, params['nn_params']['num_classes'])
acc, recall, specificity, ppv, npv = model_measurements(matrix)
accs_rf.append(acc)
measurements_rf[cv_iteration_,:] = [acc, recall, specificity, ppv, npv]
# Save models
if params['save_FLAG']:
models = {
'nn_scalers': scalers,
'pca': pca,
'transfer_models': trained_transfer,
'classifier_models': trained_classifier,
'svm_model': svm,
'rf_model': rf
}
save_models(models, params['output_dir'], str(cv_iteration_))
# Output the performance indices as xlsx
if params['save_FLAG']:
models = {
'transfer_models': measurements_transfer,
'classifier_models': measurements_classifier,
'svm_model': measurements_svm,
'rf_model': measurements_rf
}
save_result_csv(models, ref, ['acc', 'recall', 'specificity', 'ppv', 'npv'], params['output_dir'])
# Return accuracies for output
return accs_transfer, accs_classifier, accs_svm, accs_rf
def train(self, plot_period: int=5):
""" define loss-, optimzer- and scheduler-functions """
criterion_disc = nn.BCELoss()
criterion_gen = nn.BCELoss()
optimizer_disc = torch.optim.Adam(self.discriminator.parameters(), lr=self.disc_lr, betas=(0.5, 0.999))
optimizer_gen = torch.optim.Adam(self.generator.parameters(), lr=self.gen_lr, betas=(0.5, 0.999))
""" create benchmark """
self.benchmark_logger.create_entry(self.benchmark_id, optimizer_disc, criterion_disc, self.epochs, self.batch_size, self.disc_lr, self.gen_lr, self.disc_lr_decay, self.gen_lr_decay, self.lr_decay_period, self.gaussian_noise_range)
# initial noise rate
noise_rate = self.gaussian_noise_range[0]
# total loss log
loss_disc_history, loss_disc_real_history, loss_disc_fake_history = [], [], []
loss_gen_history = []
for epoch in range(self.epochs):
# epoch loss log
epoch_loss_disc, epoch_loss_disc_real, epoch_loss_disc_fake = [], [], []
epoch_loss_gen = []
for iteration in tqdm(range(self.iterations), ncols=120, desc="batch-iterations"):
images_real, targets_real, images_fake, targets_fake = self._create_batch(iteration)
""" train discriminator """
# update every third iteration to make the generator stronger
self.discriminator.zero_grad()
# train with real images
predictions_real = self.discriminator.train()(images_real, gaussian_noise_rate=noise_rate)
loss_real = criterion_disc(predictions_real, targets_real)
loss_real.backward()
# train with fake images
predictions_fake = self.discriminator.train()(images_fake, gaussian_noise_rate=noise_rate)
loss_fake = criterion_disc(predictions_fake, targets_fake)
loss_fake.backward(retain_graph=True)
if iteration % 1 == 0:
optimizer_disc.step()
# save losses
epoch_loss_disc.append(loss_real.item() + loss_fake.item())
epoch_loss_disc_real.append(loss_real.item())
epoch_loss_disc_fake.append(loss_fake.item())
""" train generator """
self.generator.zero_grad()
# train discriminator on fake images with target "real image" ([1, 0])
predictions_fake = self.discriminator.train()(images_fake)
loss_gen = criterion_gen(predictions_fake, targets_real)
loss_gen.backward()
optimizer_gen.step()
epoch_loss_gen.append(loss_gen.item())
""" linear gaussian noise decay for disc. inputs """
noise_rate = np.linspace(self.gaussian_noise_range[0], self.gaussian_noise_range[1], self.epochs)[epoch]
""" save models """
save_models(self.generator, self.discriminator, save_to=(self.models_path), current_epoch=epoch, period=5)
""" calculate average losses of the epoch """
current_loss_disc, current_loss_disc_real, current_loss_disc_fake = round(np.mean(epoch_loss_disc), 4), round(np.mean(epoch_loss_disc_real), 4), round(np.mean(epoch_loss_disc_fake), 4)
current_loss_gen = round(np.mean(epoch_loss_gen), 4)
""" get learning-rate """
current_disc_lr = round(optimizer_disc.param_groups[0]["lr"], 7)
current_gen_lr = round(optimizer_gen.param_groups[0]["lr"], 7)
""" learning-rate decay (set 'p' to 'False' for not doing lr-decay) """
do = False
if do:
optimizer_disc.param_groups[0]["lr"] = lr_decay(lr=optimizer_disc.param_groups[0]["lr"], epoch=epoch, decay_rate=self.disc_lr_decay, period=self.lr_decay_period)
optimizer_gen.param_groups[0]["lr"] = lr_decay(lr=optimizer_gen.param_groups[0]["lr"], epoch=epoch, decay_rate=self.gen_lr_decay, period=self.lr_decay_period)
""" save losses for plotting """
loss_disc_history.append(current_loss_disc); loss_disc_real_history.append(current_loss_disc_real); loss_disc_fake_history.append(current_loss_disc_fake)
loss_gen_history.append(current_loss_gen)
""" print trainings progress """
print_progress(epoch, self.epochs, current_loss_disc, current_loss_disc_real, current_loss_disc_fake, current_loss_gen, current_disc_lr, current_gen_lr)
""" plot generated images """
if plot_period is not None:
show_generated(self.generator, view_seconds=1, current_epoch=epoch, period=plot_period, save_to=(self.generated_images_path + "/" + str(epoch + 1) + ".png"))
""" plot loss history """
plot_loss_history(loss_disc_history, loss_disc_real_history, loss_disc_fake_history, loss_gen_history, save_to=(self.plots_path + "/" + self.benchmark_id + ".png"))
def training(params,
logger,
train_func=do_forward_pass,
save_model_to="",
model_def=sinno_model):
# create data loader
source_loader = get_data_loader(data_path=params.source_data_path,
params=params)
target_loader = get_data_loader(data_path=params.target_data_path,
params=params)
# model components
basic_feat_ext = model_def.BasicFeatExtractor(params=params)
code_gen = model_def.CodeGen(params=params)
discriminator = model_def.Discriminator(params=params)
# optimizers
learning_rate = params.learning_rate
opt_basic_feat = torch.optim.Adam(basic_feat_ext.parameters(),
lr=learning_rate)
opt_code_gen = torch.optim.Adam(code_gen.parameters(), lr=learning_rate)
opt_discriminator = torch.optim.Adam(discriminator.parameters(),
lr=learning_rate)
# for saving gradient
grad_records = {}
def save_grad(name):
def hook(grad):
grad_records[name] = grad
return hook
# training
code_loss_records = []
discr_loss_records = []
discr_acc_records = []
for i in range(params.iterations):
# refresh data loader
itertools.tee(source_loader)
itertools.tee(target_loader)
zipped_loader = enumerate(zip(source_loader, target_loader))
acc_code_loss, acc_discr_loss, acc_discr_acc = 0, 0, 0
logger.info("epoch {}/{} started".format(i, params.iterations))
# train using minibatches
for step, ((images_src, labels_src), (images_tgt,
labels_tgt)) in zipped_loader:
logger.info("batch {}".format(step))
# clear gradients
opt_basic_feat.zero_grad()
opt_code_gen.zero_grad()
opt_discriminator.zero_grad()
# create input variables
src_imgs = Variable(images_src).float()
tgt_imgs = Variable(images_tgt).float()
# src_lbls = Variable(labels_src)
# tgt_lbls = Variable(labels_tgt)
# call the specific training procedure to calculate loss
train_out = train_func(params=params,
basic_feat_ext=basic_feat_ext,
code_gen=code_gen,
discriminator=discriminator,
imgs=[src_imgs, tgt_imgs],
labels=[labels_src, labels_tgt])
total_loss, code_loss, discr_loss, discr_acc = train_out[
"total_loss"], train_out["code_loss"], train_out[
"discriminator_loss"], train_out["discriminator_acc"]
# do weights update
total_loss.backward()
opt_basic_feat.step()
opt_code_gen.step()
opt_discriminator.step()
acc_code_loss += code_loss.cpu().data.numpy()[0]
acc_discr_loss += discr_loss.cpu().data.numpy()[0]
acc_discr_acc += discr_acc
code_loss_records.append(acc_code_loss / (step + 1))
discr_loss_records.append(acc_discr_loss / (step + 1))
discr_acc_records.append(acc_discr_acc / (step + 1))
logger.info(
"epoch {} | code loss: {}, discr loss: {}, discr acc: {}".format(
i, acc_code_loss / (step + 1), acc_discr_loss / (step + 1),
acc_discr_acc / (step + 1)))
# save model params
save_models(models={
"basic_feat_ext": basic_feat_ext,
"code_gen": code_gen,
"discriminator": discriminator
},
save_model_to=save_model_to,
save_obj=True,
save_params=True)
# save the training settings
save_param_module(params=params,
save_to=os.path.join(save_model_to,
"train_settings.txt"))
logger.info("model saved")
return {
"code_loss_records": code_loss_records,
"discr_loss_records": discr_loss_records,
"discr_acc_records": discr_acc_records
}
g_loss.backward()
optimizer_G.step()
print(
"[{0:%Y/%m/%d %H:%M:%S}] [Epoch {1:d}/{2:d}] [Batch {3:d}/{4:d}] [D loss: {5:f}] [G loss: {6:f}]".format(datetime.datetime.now(),
epoch+1, opt.n_epochs, batch+1, n_batches, d_loss.item(), g_loss.item())
)
d_losses.append(d_losses_in_n_critic/opt.n_critic)
d_losses_in_n_critic = 0
g_losses.append(g_loss.item())
if (epoch*n_batches + batch) % opt.save_images_every_n_batches == 0:
utils.plot_images(fake_imgs, fig_dir, model_name, epoch*n_batches + batch)
utils.plot_losses(d_losses, g_losses, opt.n_critic, fig_dir, model_name)
if (epoch*n_batches + batch) % opt.save_weights_every_n_batches == 0:
utils.save_models({"gen": generator, "disc": discriminator}, model_dir, epoch*n_batches + batch,
opt.keep_last_n_weights)
except KeyboardInterrupt:
pass
# Save gen and disc
print("Saving final models...")
torch.save(generator, os.path.join(model_dir, model_name + "_generator.pth"))
torch.save(discriminator, os.path.join(model_dir, model_name + "_discriminator.pth"))
print("DONE.")
def train(model, optimizer, train_loader,valid_loader, args):
encoder, Kencoder, manager, decoder = [*model]
parameters = list(encoder.parameters()) + list(Kencoder.parameters()) + \
list(manager.parameters()) + list(decoder.parameters())
NLLLoss = nn.NLLLoss(reduction='mean', ignore_index=params.PAD)
KLDLoss = nn.KLDivLoss(reduction='batchmean')
for epoch in range(args.n_epoch):
epoch_start_time = time.time()
encoder.train(), Kencoder.train(), manager.train(), decoder.train()
b_loss = 0
k_loss = 0
n_loss = 0
t_loss = 0
print('Start training epoch %d ...'%(epoch))
for step, (src_X, src_y, src_K, tgt_y) in enumerate(train_loader):
# if step>100:
# break
src_X = src_X.cuda()
src_y = src_y.cuda()
src_K = src_K.cuda()
tgt_y = tgt_y.cuda()
optimizer.zero_grad()
encoder_outputs, hidden, x = encoder(src_X)
#Warning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead. (masked_fill__cuda at ..\aten\src\ATen\native\cuda\LegacyDefinitions.cpp:19)
#encoder_mask = (src_X == 0)[:, :encoder_outputs.size(0)].unsqueeze(1).byte()
encoder_mask = (src_X == 0)[:, :encoder_outputs.size(0)].unsqueeze(1).bool()
y = Kencoder(src_y)
K = Kencoder(src_K)
prior, posterior, k_i, k_logits = manager(x, y, K)
kldiv_loss = KLDLoss(prior, posterior.detach())
n_vocab = params.n_vocab
seq_len = src_y.size(1) - 1
k_logits = k_logits.repeat(seq_len, 1, 1).transpose(0, 1).contiguous().view(-1, n_vocab)
bow_loss = NLLLoss(k_logits, src_y[:, 1:].contiguous().view(-1))
n_batch = src_X.size(0)
max_len = tgt_y.size(1)
outputs = torch.zeros(max_len, n_batch, n_vocab).cuda()
hidden = hidden[params.n_layer:]
output = torch.LongTensor([params.SOS] * n_batch).cuda() # [n_batch]
for t in range(max_len):
output, hidden, attn_weights = decoder(output, k_i, hidden, encoder_outputs, encoder_mask)
outputs[t] = output
is_teacher = random.random() < args.tfr # teacher forcing ratio
top1 = output.data.max(1)[1]
output = tgt_y[:, t] if is_teacher else top1
outputs = outputs.transpose(0, 1).contiguous()
nll_loss = NLLLoss(outputs.view(-1, n_vocab),
tgt_y.contiguous().view(-1))
loss = kldiv_loss + nll_loss + bow_loss
loss.backward()
clip_grad_norm_(parameters, args.grad_clip)
optimizer.step()
b_loss += bow_loss.item()
k_loss += kldiv_loss.item()
n_loss += nll_loss.item()
t_loss += loss.item()
if (step + 1) % 10 == 0:
k_loss /= 10
n_loss /= 10
b_loss /= 10
t_loss /= 10
print("Epoch [%.2d/%.2d], Step [%.4d/%.4d], epoch_time:%.2f s: total_loss=%.4f kldiv_loss=%.4f bow_loss=%.4f nll_loss=%.4f"
% (epoch + 1, args.n_epoch,
step + 1, len(train_loader), time.time()-epoch_start_time,
t_loss, k_loss, b_loss, n_loss))
k_loss = 0
n_loss = 0
b_loss = 0
t_loss = 0
print('Start evaluateing epoch %d ...' % (epoch))
valid_loss = evaluate_loss(model, epoch, valid_loader)
# save models
save_models(model, params.all_restore,epoch, valid_loss)
def main():
args = parse_arguments()
n_vocab = params.n_vocab
n_layer = params.n_layer
n_hidden = params.n_hidden
n_embed = params.n_embed
n_batch = args.n_batch
temperature = params.temperature
train_path = params.train_path
#test_path = params.test_path
valid_path = params.valid_path
vocab_path = params.vocab_path
assert torch.cuda.is_available()
# print("building the vocab...")
# vocab = build_vocab_music(train_path, n_vocab)
# # save vocab
# print("saving the vocab...")
# with open(vocab_path, 'w',encoding='utf-8') as fp:
# json.dump(vocab.stoi, fp, ensure_ascii=False)
# load vocab
print("loading the vocab...")
vocab = Vocabulary()
with open(vocab_path, 'r',encoding='utf-8') as fp:
vocab.stoi = json.load(fp)
# load data and change to id
print("loading_data...")
train_X, train_y, train_K = load_data(train_path, vocab)
train_loader = get_data_loader(train_X, train_y, train_K, n_batch,True)
print("successfully loaded train data")
valid_X, valid_y, valid_K = load_data(valid_path, vocab)
valid_loader = get_data_loader(valid_X, valid_y, valid_K, n_batch,False)
print("successfully loaded valid data")
encoder = Encoder(n_vocab, n_embed, n_hidden, n_layer, vocab).cuda()
Kencoder = KnowledgeEncoder(n_vocab, n_embed, n_hidden, n_layer, vocab).cuda()
manager = Manager(n_hidden, n_vocab, temperature).cuda()
decoder = Decoder(n_vocab, n_embed, n_hidden, n_layer, vocab).cuda()
if args.restore:
encoder = init_model(encoder, restore=params.encoder_restore)
Kencoder = init_model(Kencoder, restore=params.Kencoder_restore)
manager = init_model(manager, restore=params.manager_restore)
decoder = init_model(decoder, restore=params.decoder_restore)
model = [encoder, Kencoder, manager, decoder]
parameters = list(encoder.parameters()) + list(Kencoder.parameters()) + \
list(manager.parameters()) + list(decoder.parameters())
optimizer = optim.Adam(parameters, lr=args.lr)
# pre_train knowledge manager
print("start pre-training")
pre_train(model, optimizer, train_loader,args)
print("start training")
train(model, optimizer, train_loader,valid_loader , args)
# save final model
save_models(model, params.all_restore)
def train(
dataloader,
discr,
gen,
discr_loss,
gen_loss,
discr_optimizer,
gen_optimizer,
device,
n_epochs=500,
n_discr=1,
n_gen=1,
image_interval=20,
save_interval=20,
score_interval=20,
save_dir=None,
tb_writer=None,
scheduler=None,
scorer=None,
seed=999,
verbose=True,
):
# Set random seed for reproduceability
random.seed(seed)
torch.manual_seed(seed)
step = 0
try:
with tqdm(total=n_epochs, desc=f'Progress: ', unit='epoch') as pbar:
for epoch in range(n_epochs):
for real in dataloader:
# Get real and fake data
real = real.to(device)
batch_size = real.shape[0]
fake = gen.sample(batch_size=batch_size)
# Train Discriminator
discr_optimizer.zero_grad()
D_real = discr(real)
D_fake = discr(fake.detach())
loss_discr = discr_loss(D_real, D_fake)
loss_discr.backward()
if epoch % (n_gen + n_discr) < n_discr:
discr_optimizer.step()
# Train Generator
gen_optimizer.zero_grad()
D_fake = discr(fake)
loss_gen = gen_loss(D_fake)
loss_gen.backward()
if epoch % (n_gen + n_discr) >= n_discr:
gen_optimizer.step()
is_last_epoch = (step == len(dataloader) * n_epochs - 1)
# Log Losses and Images
stats = {
'loss_D': loss_discr.item(),
'loss_G': loss_discr.item()
}
if tb_writer:
tb_writer.add_scalar(
'D_real',
torch.sigmoid(D_real).mean().item(), step)
tb_writer.add_scalar(
'D_fake',
torch.sigmoid(D_fake).mean().item(), step)
tb_writer.add_scalar('loss_D', stats['loss_D'], step)
tb_writer.add_scalar('loss_G', stats['loss_G'], step)
# Show generated images
if step % int(image_interval *
len(dataloader)) == 1 or is_last_epoch:
with torch.no_grad():
fake = gen.fixed_sample().detach().cpu()
fake = F.interpolate(fake,
scale_factor=[1, 1, 0.5, 0.5],
mode='bilinear')
fake = vutils.make_grid(fake,
padding=2,
normalize=True)[np.newaxis]
tb_writer.add_images('images', fake, step)
# Save model
if step % int(save_interval *
len(dataloader)) == 1 or is_last_epoch:
save_models(save_dir, gen, discr, step)
# Get score
if scorer:
if step % int(score_interval *
len(dataloader)) == 1 or is_last_epoch:
if verbose:
print('Calculating score')
stats['score'] = scorer.get_score(gen, step)
step += 1
# Update progress bar
pbar.set_postfix(**stats)
pbar.update()
return True
except KeyboardInterrupt:
save_models(save_dir, gen, discr, step)
return False
optimizer.step()
train_losses.append(float(loss))
val_history_Us, val_future_Us = next(iter(val_dataloader))
val_history_Xs = val_history_Us[:, :, 0].view(-1, k, 1)
inferred_U_val = predict_state(val_history_Xs)
val_loss = torch.mean(
torch.abs(inferred_U_val.detach() - val_future_Us.detach())**2)
val_losses.append(float(val_loss))
scheduler.step(val_loss)
if EPOCH % 2 == 0:
print(
"EPOCH {} finished with training loss: {} | validation loss: {} | lr: {} "
.format(EPOCH, loss, val_loss, get_lr(optimizer)))
save_models(model_dir, encoder_rnn)
save_optimizer(model_dir, optimizer)
print("TRAINING IS FINISHED!")
plt.plot(np.log(train_losses),
label='train loss'), plt.plot(np.log(val_losses),
label="validation loss")
plt.legend(), plt.show()
save_models(model_dir, encoder_rnn)
save_optimizer(model_dir, optimizer)
else:
load_models(model_dir, encoder_rnn)
load_optimizer(model_dir, optimizer)
with torch.no_grad():
def train_eval(dqn_agent, replay_buffer, env, eval_env, device, logger,
CONFIG):
"""Train and evaluate agent on given environments according to given configuration."""
# Log to TensorBoard and W&B
if CONFIG.USE_TENSORBOARD:
from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(log_dir="tensorboard_logs")
if CONFIG.USE_WANDB:
import wandb
wandb.init(project="implementations-dqn", config=CONFIG)
wandb.watch(dqn_agent.q_net)
# Check if SAVE_DIR is defined
if not CONFIG.SAVE_DIR:
logger.warning("No save directory specified: the model will be lost!")
else:
# Episodic return of saved best model
saved_model_eval_episode_return = -float("inf")
# Unique save directory to prevent rewrite
unique_save_dir = f"{CONFIG.SAVE_DIR}/{CONFIG.ENV_NAME}/{get_timestamp()}/"
# Setup epsilon decay function
get_epsilon = get_linear_anneal_func(
start_value=CONFIG.EPSILON_DECAY_START_VALUE,
end_value=CONFIG.EPSILON_DECAY_END_VALUE,
start_step=CONFIG.EPSILON_DECAY_START_STEP,
end_step=CONFIG.EPSILON_DECAY_END_STEP,
)
# Main training loop
episode_return = 0
episode_i = 0
eval_i = 0
obs = env.reset()
for step_i in range(CONFIG.ENV_STEPS + 1):
# Interact with the environment and save the experience
epsilon = get_epsilon(step_i)
action = dqn_agent.select_action(np.expand_dims(obs, 0), epsilon)
next_obs, rew, done, info = env.step(action)
replay_buffer.append(Transition(obs, action, rew, next_obs, done))
# Train QNetwork
if (step_i % CONFIG.UPDATE_FREQUENCY == 0
and len(replay_buffer) >= CONFIG.MIN_REPLAY_BUFFER_SIZE):
experiences = replay_buffer.get_torch_batch(CONFIG.BATCH_SIZE)
td_loss = dqn_agent.train(experiences, discount=CONFIG.DISCOUNT)
# Log td_loss and epsilon
if step_i % CONFIG.LOG_FREQUENCY == 0:
logger.debug(
"Episode {:4d} Steps {:5d} Epsilon {:6.6f} Loss {:6.6f}"
.format(episode_i, step_i, epsilon, td_loss))
if CONFIG.USE_TENSORBOARD:
writer.add_scalar("td_loss", td_loss, step_i)
writer.add_scalar("epsilon", epsilon, step_i)
if CONFIG.USE_WANDB:
wandb.log({
"td_loss": td_loss,
"epsilon": epsilon
},
step=step_i)
# Update target network periodically
# NOTE(seungjaeryanlee): The paper specifies frequency measured in the number
# of parameter updates, which is dependent on UPDATE_FREQUENCY. However, in
# the code the frequency is measured in number of perceived states. We follow
# the latter.
# https://github.com/deepmind/dqn/blob/9d9b1d1/dqn/NeuralQLearner.lua#L356
if step_i % CONFIG.TARGET_NET_UPDATE_FREQUENCY == 0:
dqn_agent.update_target_q_net()
# Prepare for next step
episode_return += rew
obs = next_obs
# Prepare for next episode if episode is finished
if done:
# Log episode metrics
logger.info("Episode {:4d} Steps {:5d} Return {:4d}".format(
episode_i, step_i, int(episode_return)))
if CONFIG.USE_TENSORBOARD:
writer.add_scalar("train/episode_return", episode_return,
episode_i)
if CONFIG.USE_WANDB:
wandb.log(
{
"train/episode_return": episode_return,
"train/episode_count": episode_i,
},
step=step_i,
)
# Prepare for new episode
env.reset()
episode_return = 0
episode_i += 1
# Evaluate agent periodically
if step_i % CONFIG.EVAL_FREQUENCY == 0 and CONFIG.EVAL_EPISODES > 0:
all_eval_episode_return = []
# Run multiple evaluation episodes
for _ in range(CONFIG.EVAL_EPISODES):
# Run evaluation
eval_done = False
eval_obs = eval_env.reset()
eval_episode_return = 0
while not eval_done:
eval_action = dqn_agent.select_action(
np.expand_dims(eval_obs, 0),
epsilon=CONFIG.EVAL_EPSILON)
eval_obs, eval_rew, eval_done, _ = eval_env.step(
eval_action)
eval_episode_return += eval_rew
all_eval_episode_return.append(eval_episode_return)
avg_eval_episode_return = np.mean(all_eval_episode_return)
# Log results
logger.info("EVALUATION Steps {:5d} Return {:7.2f}".format(
step_i, avg_eval_episode_return))
if CONFIG.USE_TENSORBOARD:
writer.add_scalar("eval/avg_episode_return",
avg_eval_episode_return, eval_i)
writer.add_histogram("eval/episode_returns",
np.array(all_eval_episode_return))
if CONFIG.USE_WANDB:
wandb.log(
{
"eval/avg_episode_return":
avg_eval_episode_return,
"eval/episode_returns":
wandb.Histogram(all_eval_episode_return),
"eval/episode_count":
eval_i,
},
step=step_i,
)
# Update save file if necessary
if (CONFIG.SAVE_DIR and
saved_model_eval_episode_return <= eval_episode_return):
saved_model_eval_episode_return = eval_episode_return
save_models(
unique_save_dir,
filename="best",
q_net=dqn_agent.q_net,
optimizer=dqn_agent.optimizer,
)
logger.info(f"Model succesfully saved at {unique_save_dir}")
eval_i += 1
# Save trained agent
if CONFIG.SAVE_DIR:
save_models(
unique_save_dir,
filename="last",
q_net=dqn_agent.q_net,
optimizer=dqn_agent.optimizer,
)
logger.info(f"Model successfully saved at {unique_save_dir}")
def main(batch_size,
embed_size,
num_hiddens,
num_layers,
ln_hidden,
ln_output,
rec_unit,
learning_rate=1e-4,
log_step=10,
num_epochs=50,
save_step=100,
ngpu=1):
# hyperparameters
num_workers = 0
checkpoint_dir = 'checkpoint'
# Image Preprocessing
transform = {
'train':
transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
]),
'val':
transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
]),
}
# load data
vocab = build_vocab(path='relative_captions_shoes.json')
train_data, train_loader = data_and_loader(
path='relative_captions_shoes.json',
mode='train',
vocab=vocab,
transform=transform['train'],
batch_size=batch_size)
val_data, val_loader = data_and_loader(path='relative_captions_shoes.json',
mode='valid',
vocab=vocab,
transform=transform['val'],
batch_size=batch_size)
losses_val = []
losses_train = []
# Build the models
initial_step = initial_epoch = 0
encoder = CNN(embed_size) ### embed_size: power of 2
middle = fcNet(embed_size, ln_hidden, ln_output)
decoder = RNN(ln_output,
num_hiddens,
len(vocab),
num_layers,
rec_unit=rec_unit,
drop_out=0.1)
# Loss, parameters & optimizer
loss_fun = nn.CrossEntropyLoss()
params = list(decoder.parameters()) + list(
encoder.linear.parameters()) + list(encoder.batchnorm.parameters())
optimizer = torch.optim.Adam(params, lr=learning_rate)
if torch.cuda.is_available():
encoder.cuda()
decoder.cuda()
# Train the Models
total_step = len(train_loader)
try:
for epoch in range(initial_epoch, num_epochs):
print('Epoch: {}'.format(epoch))
for step, (images, captions,
lengths) in enumerate(train_loader, start=initial_step):
# Set mini-batch dataset
images = Variable(images)
captions = Variable(captions)
targets = pack_padded_sequence(captions,
lengths,
batch_first=True)[0]
# Forward, Backward and Optimize
decoder.zero_grad()
middle.zero_grad()
encoder.zero_grad()
if ngpu > 1:
# run on multiple GPUs
features = nn.parallel.data_parallel(
encoder, images, range(ngpu))
rnn_input = nn.parallel.data_parallel(
middle, features, range(ngpu))
outputs = nn.parallel.data_parallel(
decoder, features, range(ngpu))
else:
# run on single GPU
features = encoder(images)
rnn_input = middle(features)
outputs = decoder(rnn_input, captions, lengths)
train_loss = loss_fun(outputs, targets)
losses_train.append(train_loss.item())
train_loss.backward()
optimizer.step()
# Run validation set and predict
if step % log_step == 0:
encoder.batchnorm.eval()
# run validation set
batch_loss_val = []
for val_step, (images, captions,
lengths) in enumerate(val_loader):
images = Variable(images)
captions = Variable(captions)
targets = pack_padded_sequence(captions,
lengths,
batch_first=True)[0]
#features = encoder(target_images) - encoder(refer_images)
features = encoder(images)
rnn_input = middle(features)
outputs = decoder(rnn_input, captions, lengths)
val_loss = loss_fun(outputs, targets)
batch_loss_val.append(val_loss.item())
losses_val.append(np.mean(batch_loss_val))
# predict
sampled_ids = decoder.sample(rnn_input)
sampled_ids = sampled_ids.cpu().data.numpy()[0]
sentence = utils.convert_back_to_text(sampled_ids, vocab)
print('Sample:', sentence)
true_ids = captions.cpu().data.numpy()[0]
sentence = utils.convert_back_to_text(true_ids, vocab)
print('Target:', sentence)
print(
'Epoch: {} - Step: {} - Train Loss: {} - Eval Loss: {}'
.format(epoch, step, losses_train[-1], losses_val[-1]))
encoder.batchnorm.train()
# Save the models
if (step + 1) % save_step == 0:
save_models(encoder, middle, decoder, optimizer, step,
epoch, losses_train, losses_val,
checkpoint_dir)
dump_losses(losses_train, losses_val,
os.path.join(checkpoint_dir, 'losses.pkl'))
except KeyboardInterrupt:
pass
finally:
# Do final save
utils.save_models(encoder, middle, decoder, optimizer, step, epoch,
losses_train, losses_val, checkpoint_dir)
utils.dump_losses(losses_train, losses_val,
os.path.join(checkpoint_dir, 'losses.pkl'))
def train(sess, env, env_eval, actor, critic, actor_eval, critic_eval, saver):
def train_ep(p_ev=None, ep_len=None):
t = time.time()
ep_ave_r = run_episode(env,
actor,
critic,
replay_buffer,
rho_tar_list[-1],
pol_eval=p_ev,
episode_len=ep_len)
ep_time = time.time() - t
return ep_ave_r, ep_time
def test_ep(name, plot, save):
eval_len = config.eval_episode_len
t = time.time()
r, r_tar = test_actor(env_eval,
actor_eval,
critic_eval,
rho_list,
rho_tar_list,
eval_len=eval_len,
name=name,
plot=plot,
save=save)
rho_list.append(r), rho_tar_list.append(r_tar)
ev_time = time.time() - t
return ev_time
def table_title():
logger.info(
'{:^5} | {:^9} | {:^9} | {:^5} | {:^5} | {:^11} | {:^9} | {:^9} |'.
format('Epi.', 'lr', 'Noise std', 'Epi. secs', 'Eval secs',
'Epi. Avg Ro', 'Rho', 'Rho Target'))
def table_entry(idx):
logger.info(
'{:^5d} | {:^9.4e} | {:^9.4f} | {:^9d} | {:^9d} | {:^11.05f} | {:^9.05f} | {:^10.05f} |'
.format(idx, actor.lr(), config.noise_std, int(ep_elapsed),
int(ev_elapsed), ep_ave_reward, rho_list[-1],
rho_tar_list[-1]))
# initiate replay buffer
replay_buffer = ReplayBuffer(config.buffer_size, config.seed)
# initiate arrays for documenting average cumulative reward of online and target networks.
rho_list, rho_tar_list = list(), list()
logger.info("evaluating initial average reward")
ev_elapsed = test_ep(name='pol_eval', plot=None, save=None)
logger.info("performing initial episode of policy evaluation")
ep_ave_reward, ep_elapsed = train_ep(p_ev=True, ep_len=1000)
table_title()
table_entry(-1)
for i in range(config.episode_num):
logger.debug("global step actor: {}".format(actor.global_step()))
logger.debug("global step critic: {}".format(critic.global_step()))
plot = None
save = True if i % 50 == 0 else None
ep_ave_reward, ep_elapsed = train_ep()
ev_elapsed = test_ep(name=i, plot=plot, save=save)
table_entry(i)
if save:
save_models(saver, sess, config.directory)
if i % 100 == 0:
print('long eval')
test_actor(env_eval,
actor_eval,
critic_eval,
rho_list,
rho_tar_list,
eval_len=100,
name="final",
plot=True,
verbose=True)
return rho_list, rho_tar_list
def train(
dataloader,
discr,
gen,
discr_loss,
gen_loss,
discr_optimizer,
gen_optimizer,
tb_writer,
device,
n_epochs = 500,
n_discr = 1,
n_gen = 1,
image_interval = 1,
save_interval = 1,
save_dir = None,
scheduler = None,
seed = 999,
):
random.seed(seed)
torch.manual_seed(seed)
step = 0
try:
with tqdm(total=n_epochs, desc=f'Progress: ',unit='epoch') as pbar:
for epoch in range(n_epochs):
for real in dataloader:
# Get real and fake data
real = real.to(device)
batch_size = real.shape[0]
fake = gen.sample(batch_size = batch_size)
# Train Discriminator
discr_optimizer.zero_grad()
D_real = discr(real)
D_fake = discr(fake.detach())
loss_discr = discr_loss(D_real,D_fake)
loss_discr.backward()
if epoch % (n_gen + n_discr) < n_discr:
discr_optimizer.step()
# Train Generator
gen_optimizer.zero_grad()
D_fake = discr(fake)
loss_gen = gen_loss(D_fake)
loss_gen.backward()
if epoch % (n_gen + n_discr) >= n_discr:
gen_optimizer.step()
# Log Losses
tb_writer.add_scalar('D_real',torch.sigmoid(D_real).mean().item(),step)
tb_writer.add_scalar('D_fake',torch.sigmoid(D_fake).mean().item(),step)
tb_writer.add_scalar('loss_D',loss_discr.item(),step)
tb_writer.add_scalar('loss_G',loss_gen.item(),step)
# Show generated images
if step % int(image_interval*len(dataloader)) == 1 or (step == len(dataloader)*n_epochs - 1):
with torch.no_grad():
fake = gen.fixed_sample().detach().cpu()
grid = vutils.make_grid(fake, padding=2, normalize=True)[np.newaxis]
tb_writer.add_images('images', grid, step)
if step % int(save_interval*len(dataloader)) == 1 or (step == len(dataloader)*n_epochs - 1):
save_models(save_dir,gen,discr,step)
step += 1
pbar.set_postfix(**{'loss_D':loss_discr.item(),'loss_G':loss_discr.item()})
pbar.update()
return True
except KeyboardInterrupt:
save_models(save_dir,gen,discr,step)
return False
criterion=criterion,
conditional_gen=False)
generative_trainer(loader_mix=loader_mix,
train_loader=loader2,
generator=generator2,
EP=EP,
arguments=arguments,
criterion=criterion,
conditional_gen=False)
# save models
savepath = os.path.join(os.getcwd(), 'model_parameters')
if not os.path.exists(savepath):
os.mkdir(savepath)
ut.save_models([generator1, generator2], [discriminator1, discriminator2],
exp_info, savepath, arguments)
#check1 = generator1.parameters().next()
#print('Sum of generator1 parameters is:', check1.sum())
#
#check2 = generator2.parameters().next()
#print('Sum of generator2 parameters is:', check2.sum())
###
# Separate out the sources
if arguments.task == 'mnist':
maxlikelihood_separatesources(
generators=[generator1, generator2],
discriminators=[discriminator1, discriminator2],
loader_mix=loader_mix,
schema_enc_outputs, schema_enc_hidden = encoder_dict['SchemaEncoder'](schema.t())
schema_enc_outputs = schema_enc_outputs.to(device)
schema_enc_hidden = schema_enc_hidden.to(device)
hidden = torch.cat((question_enc_hidden, schema_enc_hidden), dim=-1)
enc_outputs = torch.cat((question_enc_outputs, schema_enc_outputs), dim=0)
if n in [0, int(len(train_loader) / 3), 2 * int(len(train_loader) / 3)]:
print_outputs = True
print('\n\nText: {}\nSQL: {}'.format(' '.join(list(map(lambda i: idx2word[i.item()], text[0]))),
' '.join(list(map(lambda i: idx2word[i.item()], sql[0])))))
else:
print_outputs = False
# sql, idx, h, encoder_outputs, decoder_dict, current_decoder, predictions, print_outputs
outputs, targets, decoders = decoder_dict['RootDecoder'](sql, 0, hidden, enc_outputs, decoder_dict, 'RootDecoder', print_output=print_outputs)
epoch_loss = 0
for output, target, decoder_name in zip(outputs, targets, decoders):
zero_all_grads(encoder_optimizer, decoder_optimizer)
epoch_loss += loss(output, target)
valid_loss += epoch_loss.double()
print('Epoch: {} total_cost = {} validLoss = {}'.format(epoch, total_cost, valid_loss))
save_models(encoder_dict, decoder_dict, epoch)
losses.append([total_cost, valid_loss])
plt.plot(losses)
plt.show()
