def _train(idx, model, datasets, name, split, logger):
es = es_tolerance
best_eval = 100000
_eval = 100001
for epoch in range(100000):
train_gen, (val_X, val_y), steps_per_epoch = datasets.kfold(idx, split)
history = model.fit_generator(
generator=train_gen,
steps_per_epoch=steps_per_epoch,
callbacks=[TensorBoard(prefix + 'results/%s/tflog' % name)],
verbose=0)
logger.warning('[%s_%d: Epoch %d] %f' %
(name, idx, epoch, history.history['loss'][0]))
if epoch % eval_epoch == 0:
eval_results = eval_model(model, val_X, val_y)
logger.warning('####################')
logger.warning('########eval########')
for k, v in eval_results.items():
logger.warning('%s: %f' % (k, v))
logger.warning('####################')
logger.warning('####################')
_eval = eval_results['rmse']
if _eval < best_eval:
es = es_tolerance
# model.save(prefix + 'results/%s/%d.model' % (name, epoch))
elif es > 0:
es -= 1
else:
break
eval_results = eval_model(model, val_X, val_y)
return eval_results
def evaluate(self, data, scoring='f1', eval_batch_size=8):
"""
Evaluate the model on given data
:param data: Data to evaluate on
:param scoring: Scoring method to report performance with. Can be a list of multiple metrics.
:param eval_batch_size: Batch size of the evaluation loop
:return: If scoring is not a list, returns the performance. If it is a list, returns a list of performances
according to the metrics in the scoring list.
"""
self.model.to(self.device)
eval_data = TensorDataset(*data)
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data,
sampler=eval_sampler,
batch_size=eval_batch_size)
metrics = {
'f1': mt.f1_score,
'accuracy': mt.accuracy_score,
'recall': mt.recall_score,
'precision': mt.precision_score
}
if type(scoring) == str:
results = ev.eval_model(self.model, eval_dataloader,
[metrics[scoring]], self.device)
return results[0]
else:
results = ev.eval_model(self.model, eval_dataloader,
[metrics[s] for s in scoring], self.device)
return results
def main():
for epoch in range(args.n_epoch):
print('\n\n-------------------------------------------')
print('Epoch-{}'.format(epoch))
print('-------------------------------------------')
model.train()
train_iter = tqdm(enumerate(dataset.train_iter()))
train_iter.set_description_str('Training')
for it, mb in train_iter:
output = model(mb.context, mb.response)
loss = F.binary_cross_entropy_with_logits(output, mb.label)
loss.backward()
# clip_gradient_threshold(model, -10, 10)
solver.step()
solver.zero_grad()
if it > 0 and it % 1000 == 0:
# Validation
recall_at_ks = eval_model(model, dataset.valid_iter(),
max_seq_len, max_seq_len, args.gpu)
print(
'Loss: {:.3f}; recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}'
.format(loss.data[0], recall_at_ks[0], recall_at_ks[1],
recall_at_ks[4]))
save_model(model, 'ccn_lstm')
def eval_test():
print('\n\nEvaluating on test set...')
print('-------------------------------')
recall_at_ks = eval_model(model, dataset.test_iter(), args.max_context_len, args.max_response_len, args.gpu)
print('Recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}'
.format(recall_at_ks[0], recall_at_ks[1], recall_at_ks[2]))
def _train(idx, model, datasets, name, split, logger):
es = es_tolerance
best_eval = 100000
_eval = 100001
steps = 0
for epoch in range(100000):
train_gen, (val_X, val_y), steps_per_epoch = datasets.kfold(idx, split)
history = model.fit_generator(generator=train_gen, steps_per_epoch=steps_per_epoch, verbose=1)
train_loss = history.history['loss'][0]
logger.warning('[%s_%d|Train|Epoch %d|Steps %d] Loss: %f' % (name, idx, epoch, steps, train_loss))
eval_results = eval_model(model, val_X, val_y)
logger.warning('####################')
logger.warning('[%s_%d|Valid|Epoch %d|Steps %d]' % (name, idx, epoch, steps))
for k, v in eval_results.items():
logger.warning('%s: %f' % (k, v))
logger.warning('####################')
_eval = eval_results['rmse']
steps += 256
# _losses = []
# for _ in range(steps_per_epoch):
# try:
# history = model.fit_generator(generator=train_gen, steps_per_epoch=10, verbose=2)
# except StopIteration:
# train_gen, (val_X, val_y), steps_per_epoch = datasets.kfold(idx, split)
# print(steps, _)
# _losses.append(history.history['loss'][0])
#
# if steps % log_steps == 0:
# logger.warning('[%s_%d|Epoch %d|Steps %d] Loss: %f' % (name, idx, epoch, steps, np.mean(_losses)))
#
# if steps % eval_steps == 0:
# eval_results = eval_model(model, val_X, val_y)
# logger.warning('####################')
# logger.warning('####################')
# logger.warning('Epoch: %d, Steps: %d' % (epoch, steps))
# for k, v in eval_results.items():
# logger.warning('%s: %f' % (k, v))
# logger.warning('####################')
# logger.warning('####################')
# _eval = eval_results['rmse']
#
# steps += 1
if _eval < best_eval:
es = es_tolerance
# model.save(prefix + 'results/%s/%d.model' % (name, epoch))
elif es > 0:
es -= 1
else:
break
# eval_results = eval_model(model, val_X, val_y)
return eval_results
def eval_test(model):
'''
Evaluation
:param model:
:return:
'''
print('\n\nEvaluating on test set...')
print('-------------------------------')
print('Loading the best model........')
model = load_model(model, model_name)
model.eval()
recall_at_ks = eval_model(model,
udc,
'test',
gpu=args.gpu,
no_tqdm=args.no_tqdm)
print('Recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}'.format(
recall_at_ks[0], recall_at_ks[1], recall_at_ks[4]))
def main():
for epoch in range(args.n_epoch):
print('\n\n-------------------------------------------')
print('Epoch-{}'.format(epoch))
print('-------------------------------------------')
model.train()
train_iter = enumerate(dataset.train_iter())
if not args.no_tqdm:
train_iter = tqdm(train_iter)
train_iter.set_description_str('Training')
for it, mb in train_iter:
# Truncate input
#print (mb.context.lengths, mb.context)
context = mb.context[:, :args.max_context_len]
response = mb.response[:, :args.max_response_len]
#print (context)
output = model(context, response)
loss = F.binary_cross_entropy_with_logits(output, mb.label)
loss.backward()
clip_gradient_threshold(model, -10, 10)
solver.step()
solver.zero_grad()
if it > 0 and it % 1000 == 0:
# Validation
recall_at_ks = eval_model(model, dataset.valid_iter(), args.max_context_len, args.max_response_len, args.gpu, args.no_tqdm)
print('Loss: {:.3f}; recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}'
.format(loss.data[0], recall_at_ks[0], recall_at_ks[1], recall_at_ks[4]))
save_model(model, 'ccn_lstm')
from corpus import get_data, get_lexicon
from hmm import get_HMM
from viterbi import decode
from evaluation import eval_model
import random
train_set = get_data("corpus.xml")[:]
lexicon = get_lexicon("lexicon.txt")
model = get_HMM(train_set, lexicon)
results = eval_model(train_set, model)
print("\nworst_tags\n", results["worst_tags"])
print("\nworst_words:\n", results["worst_words"])
print("\nacc: ", results["accuracy"])
#print("confusion matrix:")
#print(results["confusion"].tabulate())
from corpus import get_data, get_lexicon
from hmm import get_HMM
from viterbi import decode
from evaluation import eval_model
import random
import csv
data = random.shuffle(get_data("corpus.xml"))
train_set = get_data("corpus.xml")[:3000]
test_set = get_data("corpus.xml")[3000:]
lexicon = get_lexicon("lexicon.txt")
model = get_HMM(train_set, lexicon)
results = eval_model(train_set[:1500], model)
conditions = results["worst_tags"][:10]
samples = results["worst_codes"][:10]
print(results["confusion"].tabulate(conditions, samples))
"""
import pandas as pd
tmp = pd.DataFrame(results["confusion"]).fillna(0)
#print(results["confusion"].items())
print(tmp)
"""
"""
print("train set: 3000")
print("sent acc: ",results["sent_accuracy"])
print(" acc: ",results["accuracy"])
results = eval_model(test_set, model)
def run_model():
"""
Training method
:return:
"""
best_val = 0.0
recall1s = []
for epoch in range(args.n_epoch):
print('\n\n-------------------------------------------')
print('Epoch-{}'.format(epoch))
print('-------------------------------------------')
model.train()
train_iter = enumerate(udc.get_iter('train'))
if not args.no_tqdm:
train_iter = tqdm(train_iter)
train_iter.set_description_str('Training')
train_iter.total = udc.n_train // udc.batch_size
for it, mb in train_iter:
context, response, y, cm, rm, key_r, key_mask_r = mb
output = model(
context, response, cm, rm, key_r, key_mask_r
) # Appropriate this line while running different models
loss = F.binary_cross_entropy_with_logits(output, y)
loss.backward()
solver.step()
solver.zero_grad()
# Validation
recall_at_ks = eval_model(model,
udc,
'valid',
gpu=args.gpu,
no_tqdm=args.no_tqdm)
print(
'Loss: {:.3f}; recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}'
.format(loss.data[0], recall_at_ks[0], recall_at_ks[1],
recall_at_ks[4]))
recall_1 = recall_at_ks[0]
# if epoch > 10:
# eval_test()
if best_val == 0.0:
save_model(model, model_name)
best_val = recall_1
recall1s.append(recall_1)
else:
if recall_1 > best_val:
best_val = recall_1
print("Saving model for recall@1:" + str(recall_1))
save_model(model, model_name)
else:
print("Not saving, best accuracy so far:" + str(best_val))
#Early stopping
if recall_1 < np.max(recall1s[-args.early_stop:]):
break
def on_epoch_end(self, epoch, logs=None):
tmp_score = eval_model(dev_data, self.model, max_len)[1]
if self.best_score is None or tmp_score > self.best_score:
self.best_score = tmp_score
self.model.save(os.path.join(self.model_saved_path, "best_weights"), overwrite=True)
projectPath = './param/outputModelWeights/{}'.format(projectName)
if not os.path.isdir(projectPath): os.makedirs(projectPath)
resultPath = projectPath + '/{}/'.format(now)
os.makedirs(resultPath)
callbacks = [
SaveModelBestCheckpoint(resultPath),
SaveModelLastCheckpoint(resultPath),
CSVLogger(resultPath + 'training.log'),
]
model.fit(
x_train, y_train,
epochs=epochs, batch_size=batch_size, callbacks=callbacks
)
print(resultPath)
# best_model = pathOutputModelWeights
model.load_weights(resultPath + "best_weights")
eval_model(test_data, model, max_len)
std_opt.step()
std_opt.zero_grad()
if global_step % 10 == 0:
loss = np.mean(loss_rate)
accuracy = round(correct / (10 * choices[0].shape[0]), 4)
print('[INFO]step {}: accuracy = {}, loss = {}'.format(
global_step, accuracy, loss))
painter.update_data("Train loss", [global_step], [loss])
painter.update_data("Train accuracy", [global_step],
[accuracy])
correct, loss_rate = 0, []
if global_step % 50 == 0:
test_epoch += 1
dev_acc = eval_model(model, batch_loader.get_dev_iter(),
devset_len, args.model)
test_acc = eval_model(model, batch_loader.get_test_iter(),
testset_len, args.model)
print(
'epoch {} : dev accuracy = {}, test accuracy = {}'.format(
test_epoch, dev_acc, test_acc))
painter.update_data("accuracy", [[test_epoch, test_epoch]],
[[dev_acc, test_acc]])
if test_acc > best_test_accuracy:
best_test_accuracy = test_acc
torch.save(
model, 'ckpt/test-' + args.model + '-' +
time.strftime("%H:%M:%S", start_time) + '-bestacc.pb')
def fit(models, dataset, fake_material, data, num_epochs, matcher_epochs, device, with_generator = True, just_train_classifier = False):
matcher_epochs = matcher_epochs + 1
train_loader, valid_loader = data
netD, netG = models
if just_train_classifier == True:
path = "/ctm-hdd-pool01/afpstudents/jaf/LIVEGEN_" + dataset + "_material" + str(fake_material) + "_" + str(500) + "epochs_"
netG.load_state_dict(torch.load(path + 'Generator.pth'))
model_path = "/ctm-hdd-pool01/afpstudents/jaf/LIVEGEN_" + dataset + "_material" + str(fake_material) + "_" + str(num_epochs) + "epochs_"
output_path = f"results/{DATASET}/{DATASET}_{TOA}_material{PAI}_{EPOCHS}epochs_"
# Start training
train_history = {'train_c_loss': [], 'train_g_loss': [], 'train_acc': [], 'val_c_loss': [], 'val_g_loss': [], 'val_acc': []}
netD.apply(normal_weights_init)
if just_train_classifier == False:
netG.apply(normal_weights_init)
optimizerD = optim.Adam(netD.parameters(), lr=LEARNING_RATE)
optimizerG = optim.Adam(netG.parameters(), lr=LEARNING_RATE)
netD.train()
netG.train()
score_matcher = 0
n_batches = 0
for epoch in range(num_epochs):
print("\n")
g_loss = []
d_loss = []
d_real = []
d_fake = []
for i, (x,y) in enumerate(train_loader, 0):
# (1) Update D network
## Train with all-real batch
netD.zero_grad()
netG.zero_grad()
x = x.to(device)
x_real = x[y == 0]
x_fake = x[y == 1]
if x_fake.shape[0] < 5:
continue
#b_size = real.size(0)
output_real = netD(x_real)
D_x = sigmoid(output_real.mean()).item()
real_label = torch.zeros_like(output_real, device=device)
errD_real = loss(output_real, real_label)
## Train with all-fake batch
if with_generator:
index = [i for i in range(x_fake.shape[0])]
index_to_modify = random.sample(range(x_fake.shape[0]), x_fake.shape[0]//2)
index_to_maintain = [i for i in index if i not in index_to_modify]
x_fake_to_modify = x_fake[index_to_modify,...].clone().detach()
x_fake_to_maintain = x_fake[index_to_maintain,...].clone().detach()
x_fake_modified = netG(x_fake_to_modify)
x_fake = torch.cat([x_fake_to_maintain, x_fake_modified], dim=0)
if epoch >= num_epochs - matcher_epochs and just_train_classifier == False:
try:
m_score = MATCHER(x_fake_to_modify, x_fake_modified)
except:
m_score = 10.0
if epoch == num_epochs-1:
try:
score_matcher = score_matcher + MATCHER(x_fake_to_modify, x_fake_modified)
except:
score_matcher = score_matcher + 10.0
n_batches = n_batches + 1
output_fake = netD(x_fake.detach())
D_G_z = sigmoid(output_fake.mean()).item()
fake_label = torch.ones_like(output_fake, device=device)
errD_fake = loss(output_fake, fake_label)
errD = errD_real + errD_fake
errD.backward()
optimizerD.step()
# (2) Update G network
if with_generator:
netD.zero_grad()
netG.zero_grad()
output_fake = netD(x_fake)
real_label = torch.zeros_like(output_fake, device=device)
errG = loss(output_fake, real_label)
if epoch >= num_epochs - matcher_epochs and just_train_classifier == False:
errG = errG + m_score
if just_train_classifier == False:
errG.backward()
optimizerG.step()
#######################################################################
#######################################################################
sys.stdout.write("\r" + 'EPOCH [{}/{}] ..... {}-th batch: D_real = {:.3f} | D_fake = {:.3f}'.format(epoch+1, num_epochs, i+1, D_x, D_G_z))
#Progress with fixed noise
if with_generator and just_train_classifier == False:
with torch.no_grad():
x_fake_modified = netG(x_fake_to_modify)
save_images(x_fake_to_modify[:3], x_fake_modified[:3], dataset, fake_material, epoch)
tr_c_loss, tr_g_loss, tr_acc = eval_model((netD, netG), train_loader, device, epoch, num_epochs, matcher_epochs, with_generator = with_generator)
train_history['train_c_loss'].append(tr_c_loss.item())
train_history['train_g_loss'].append(tr_g_loss.item())
train_history['train_acc'].append(tr_acc)
val_c_loss, val_g_loss, val_acc = eval_model((netD, netG), valid_loader, device, epoch, num_epochs, matcher_epochs, with_generator = with_generator)
train_history['val_c_loss'].append(val_c_loss.item())
train_history['val_g_loss'].append(val_g_loss.item())
train_history['val_acc'].append(val_acc)
# display the training loss
print()
print( '\n>> Train: C_loss = {:.3f} |'.format(tr_c_loss.item()) + ' G_loss = {:.3f} |'.format(tr_g_loss.item()) + ' Acc = {:.3f}'.format(tr_acc) )
print( '\n>> Valid: C_loss = {:.3f} |'.format(val_c_loss.item()) + ' G_loss = {:.3f} |'.format(val_g_loss.item()) + ' Acc = {:.3f}'.format(val_acc) )
print()
if epoch == num_epochs-1 and with_generator and just_train_classifier == False:
score_matcher = score_matcher / n_batches
print('\n>> Average matching score = {:.3f}'.format(score_matcher))
# save train/valid history
plot_fn = output_path + 'LIVEGEN_history.png'
plot_train_history(train_history, plot_fn=plot_fn)
#load last model
torch.save(netD.state_dict(), model_path + 'Discriminator.pth')
if just_train_classifier == False:
torch.save(netG.state_dict(), model_path + 'Generator.pth')
return (netD, train_history)
