def _train(idx, model, datasets, name, split, logger): es = es_tolerance best_eval = 100000 _eval = 100001 for epoch in range(100000): train_gen, (val_X, val_y), steps_per_epoch = datasets.kfold(idx, split) history = model.fit_generator( generator=train_gen, steps_per_epoch=steps_per_epoch, callbacks=[TensorBoard(prefix + 'results/%s/tflog' % name)], verbose=0) logger.warning('[%s_%d: Epoch %d] %f' % (name, idx, epoch, history.history['loss'][0])) if epoch % eval_epoch == 0: eval_results = eval_model(model, val_X, val_y) logger.warning('####################') logger.warning('########eval########') for k, v in eval_results.items(): logger.warning('%s: %f' % (k, v)) logger.warning('####################') logger.warning('####################') _eval = eval_results['rmse'] if _eval < best_eval: es = es_tolerance # model.save(prefix + 'results/%s/%d.model' % (name, epoch)) elif es > 0: es -= 1 else: break eval_results = eval_model(model, val_X, val_y) return eval_results
def evaluate(self, data, scoring='f1', eval_batch_size=8): """ Evaluate the model on given data :param data: Data to evaluate on :param scoring: Scoring method to report performance with. Can be a list of multiple metrics. :param eval_batch_size: Batch size of the evaluation loop :return: If scoring is not a list, returns the performance. If it is a list, returns a list of performances according to the metrics in the scoring list. """ self.model.to(self.device) eval_data = TensorDataset(*data) eval_sampler = SequentialSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=eval_batch_size) metrics = { 'f1': mt.f1_score, 'accuracy': mt.accuracy_score, 'recall': mt.recall_score, 'precision': mt.precision_score } if type(scoring) == str: results = ev.eval_model(self.model, eval_dataloader, [metrics[scoring]], self.device) return results[0] else: results = ev.eval_model(self.model, eval_dataloader, [metrics[s] for s in scoring], self.device) return results
def main(): for epoch in range(args.n_epoch): print('\n\n-------------------------------------------') print('Epoch-{}'.format(epoch)) print('-------------------------------------------') model.train() train_iter = tqdm(enumerate(dataset.train_iter())) train_iter.set_description_str('Training') for it, mb in train_iter: output = model(mb.context, mb.response) loss = F.binary_cross_entropy_with_logits(output, mb.label) loss.backward() # clip_gradient_threshold(model, -10, 10) solver.step() solver.zero_grad() if it > 0 and it % 1000 == 0: # Validation recall_at_ks = eval_model(model, dataset.valid_iter(), max_seq_len, max_seq_len, args.gpu) print( 'Loss: {:.3f}; recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}' .format(loss.data[0], recall_at_ks[0], recall_at_ks[1], recall_at_ks[4])) save_model(model, 'ccn_lstm')
def eval_test(): print('\n\nEvaluating on test set...') print('-------------------------------') recall_at_ks = eval_model(model, dataset.test_iter(), args.max_context_len, args.max_response_len, args.gpu) print('Recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}' .format(recall_at_ks[0], recall_at_ks[1], recall_at_ks[2]))
def _train(idx, model, datasets, name, split, logger): es = es_tolerance best_eval = 100000 _eval = 100001 steps = 0 for epoch in range(100000): train_gen, (val_X, val_y), steps_per_epoch = datasets.kfold(idx, split) history = model.fit_generator(generator=train_gen, steps_per_epoch=steps_per_epoch, verbose=1) train_loss = history.history['loss'][0] logger.warning('[%s_%d|Train|Epoch %d|Steps %d] Loss: %f' % (name, idx, epoch, steps, train_loss)) eval_results = eval_model(model, val_X, val_y) logger.warning('####################') logger.warning('[%s_%d|Valid|Epoch %d|Steps %d]' % (name, idx, epoch, steps)) for k, v in eval_results.items(): logger.warning('%s: %f' % (k, v)) logger.warning('####################') _eval = eval_results['rmse'] steps += 256 # _losses = [] # for _ in range(steps_per_epoch): # try: # history = model.fit_generator(generator=train_gen, steps_per_epoch=10, verbose=2) # except StopIteration: # train_gen, (val_X, val_y), steps_per_epoch = datasets.kfold(idx, split) # print(steps, _) # _losses.append(history.history['loss'][0]) # # if steps % log_steps == 0: # logger.warning('[%s_%d|Epoch %d|Steps %d] Loss: %f' % (name, idx, epoch, steps, np.mean(_losses))) # # if steps % eval_steps == 0: # eval_results = eval_model(model, val_X, val_y) # logger.warning('####################') # logger.warning('####################') # logger.warning('Epoch: %d, Steps: %d' % (epoch, steps)) # for k, v in eval_results.items(): # logger.warning('%s: %f' % (k, v)) # logger.warning('####################') # logger.warning('####################') # _eval = eval_results['rmse'] # # steps += 1 if _eval < best_eval: es = es_tolerance # model.save(prefix + 'results/%s/%d.model' % (name, epoch)) elif es > 0: es -= 1 else: break # eval_results = eval_model(model, val_X, val_y) return eval_results
def eval_test(model): ''' Evaluation :param model: :return: ''' print('\n\nEvaluating on test set...') print('-------------------------------') print('Loading the best model........') model = load_model(model, model_name) model.eval() recall_at_ks = eval_model(model, udc, 'test', gpu=args.gpu, no_tqdm=args.no_tqdm) print('Recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}'.format( recall_at_ks[0], recall_at_ks[1], recall_at_ks[4]))
def main(): for epoch in range(args.n_epoch): print('\n\n-------------------------------------------') print('Epoch-{}'.format(epoch)) print('-------------------------------------------') model.train() train_iter = enumerate(dataset.train_iter()) if not args.no_tqdm: train_iter = tqdm(train_iter) train_iter.set_description_str('Training') for it, mb in train_iter: # Truncate input #print (mb.context.lengths, mb.context) context = mb.context[:, :args.max_context_len] response = mb.response[:, :args.max_response_len] #print (context) output = model(context, response) loss = F.binary_cross_entropy_with_logits(output, mb.label) loss.backward() clip_gradient_threshold(model, -10, 10) solver.step() solver.zero_grad() if it > 0 and it % 1000 == 0: # Validation recall_at_ks = eval_model(model, dataset.valid_iter(), args.max_context_len, args.max_response_len, args.gpu, args.no_tqdm) print('Loss: {:.3f}; recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}' .format(loss.data[0], recall_at_ks[0], recall_at_ks[1], recall_at_ks[4])) save_model(model, 'ccn_lstm')
from corpus import get_data, get_lexicon from hmm import get_HMM from viterbi import decode from evaluation import eval_model import random train_set = get_data("corpus.xml")[:] lexicon = get_lexicon("lexicon.txt") model = get_HMM(train_set, lexicon) results = eval_model(train_set, model) print("\nworst_tags\n", results["worst_tags"]) print("\nworst_words:\n", results["worst_words"]) print("\nacc: ", results["accuracy"]) #print("confusion matrix:") #print(results["confusion"].tabulate())
from corpus import get_data, get_lexicon from hmm import get_HMM from viterbi import decode from evaluation import eval_model import random import csv data = random.shuffle(get_data("corpus.xml")) train_set = get_data("corpus.xml")[:3000] test_set = get_data("corpus.xml")[3000:] lexicon = get_lexicon("lexicon.txt") model = get_HMM(train_set, lexicon) results = eval_model(train_set[:1500], model) conditions = results["worst_tags"][:10] samples = results["worst_codes"][:10] print(results["confusion"].tabulate(conditions, samples)) """ import pandas as pd tmp = pd.DataFrame(results["confusion"]).fillna(0) #print(results["confusion"].items()) print(tmp) """ """ print("train set: 3000") print("sent acc: ",results["sent_accuracy"]) print(" acc: ",results["accuracy"]) results = eval_model(test_set, model)
def run_model(): """ Training method :return: """ best_val = 0.0 recall1s = [] for epoch in range(args.n_epoch): print('\n\n-------------------------------------------') print('Epoch-{}'.format(epoch)) print('-------------------------------------------') model.train() train_iter = enumerate(udc.get_iter('train')) if not args.no_tqdm: train_iter = tqdm(train_iter) train_iter.set_description_str('Training') train_iter.total = udc.n_train // udc.batch_size for it, mb in train_iter: context, response, y, cm, rm, key_r, key_mask_r = mb output = model( context, response, cm, rm, key_r, key_mask_r ) # Appropriate this line while running different models loss = F.binary_cross_entropy_with_logits(output, y) loss.backward() solver.step() solver.zero_grad() # Validation recall_at_ks = eval_model(model, udc, 'valid', gpu=args.gpu, no_tqdm=args.no_tqdm) print( 'Loss: {:.3f}; recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}' .format(loss.data[0], recall_at_ks[0], recall_at_ks[1], recall_at_ks[4])) recall_1 = recall_at_ks[0] # if epoch > 10: # eval_test() if best_val == 0.0: save_model(model, model_name) best_val = recall_1 recall1s.append(recall_1) else: if recall_1 > best_val: best_val = recall_1 print("Saving model for recall@1:" + str(recall_1)) save_model(model, model_name) else: print("Not saving, best accuracy so far:" + str(best_val)) #Early stopping if recall_1 < np.max(recall1s[-args.early_stop:]): break
def on_epoch_end(self, epoch, logs=None): tmp_score = eval_model(dev_data, self.model, max_len)[1] if self.best_score is None or tmp_score > self.best_score: self.best_score = tmp_score self.model.save(os.path.join(self.model_saved_path, "best_weights"), overwrite=True)
projectPath = './param/outputModelWeights/{}'.format(projectName) if not os.path.isdir(projectPath): os.makedirs(projectPath) resultPath = projectPath + '/{}/'.format(now) os.makedirs(resultPath) callbacks = [ SaveModelBestCheckpoint(resultPath), SaveModelLastCheckpoint(resultPath), CSVLogger(resultPath + 'training.log'), ] model.fit( x_train, y_train, epochs=epochs, batch_size=batch_size, callbacks=callbacks ) print(resultPath) # best_model = pathOutputModelWeights model.load_weights(resultPath + "best_weights") eval_model(test_data, model, max_len)
std_opt.step() std_opt.zero_grad() if global_step % 10 == 0: loss = np.mean(loss_rate) accuracy = round(correct / (10 * choices[0].shape[0]), 4) print('[INFO]step {}: accuracy = {}, loss = {}'.format( global_step, accuracy, loss)) painter.update_data("Train loss", [global_step], [loss]) painter.update_data("Train accuracy", [global_step], [accuracy]) correct, loss_rate = 0, [] if global_step % 50 == 0: test_epoch += 1 dev_acc = eval_model(model, batch_loader.get_dev_iter(), devset_len, args.model) test_acc = eval_model(model, batch_loader.get_test_iter(), testset_len, args.model) print( 'epoch {} : dev accuracy = {}, test accuracy = {}'.format( test_epoch, dev_acc, test_acc)) painter.update_data("accuracy", [[test_epoch, test_epoch]], [[dev_acc, test_acc]]) if test_acc > best_test_accuracy: best_test_accuracy = test_acc torch.save( model, 'ckpt/test-' + args.model + '-' + time.strftime("%H:%M:%S", start_time) + '-bestacc.pb')
def fit(models, dataset, fake_material, data, num_epochs, matcher_epochs, device, with_generator = True, just_train_classifier = False): matcher_epochs = matcher_epochs + 1 train_loader, valid_loader = data netD, netG = models if just_train_classifier == True: path = "/ctm-hdd-pool01/afpstudents/jaf/LIVEGEN_" + dataset + "_material" + str(fake_material) + "_" + str(500) + "epochs_" netG.load_state_dict(torch.load(path + 'Generator.pth')) model_path = "/ctm-hdd-pool01/afpstudents/jaf/LIVEGEN_" + dataset + "_material" + str(fake_material) + "_" + str(num_epochs) + "epochs_" output_path = f"results/{DATASET}/{DATASET}_{TOA}_material{PAI}_{EPOCHS}epochs_" # Start training train_history = {'train_c_loss': [], 'train_g_loss': [], 'train_acc': [], 'val_c_loss': [], 'val_g_loss': [], 'val_acc': []} netD.apply(normal_weights_init) if just_train_classifier == False: netG.apply(normal_weights_init) optimizerD = optim.Adam(netD.parameters(), lr=LEARNING_RATE) optimizerG = optim.Adam(netG.parameters(), lr=LEARNING_RATE) netD.train() netG.train() score_matcher = 0 n_batches = 0 for epoch in range(num_epochs): print("\n") g_loss = [] d_loss = [] d_real = [] d_fake = [] for i, (x,y) in enumerate(train_loader, 0): # (1) Update D network ## Train with all-real batch netD.zero_grad() netG.zero_grad() x = x.to(device) x_real = x[y == 0] x_fake = x[y == 1] if x_fake.shape[0] < 5: continue #b_size = real.size(0) output_real = netD(x_real) D_x = sigmoid(output_real.mean()).item() real_label = torch.zeros_like(output_real, device=device) errD_real = loss(output_real, real_label) ## Train with all-fake batch if with_generator: index = [i for i in range(x_fake.shape[0])] index_to_modify = random.sample(range(x_fake.shape[0]), x_fake.shape[0]//2) index_to_maintain = [i for i in index if i not in index_to_modify] x_fake_to_modify = x_fake[index_to_modify,...].clone().detach() x_fake_to_maintain = x_fake[index_to_maintain,...].clone().detach() x_fake_modified = netG(x_fake_to_modify) x_fake = torch.cat([x_fake_to_maintain, x_fake_modified], dim=0) if epoch >= num_epochs - matcher_epochs and just_train_classifier == False: try: m_score = MATCHER(x_fake_to_modify, x_fake_modified) except: m_score = 10.0 if epoch == num_epochs-1: try: score_matcher = score_matcher + MATCHER(x_fake_to_modify, x_fake_modified) except: score_matcher = score_matcher + 10.0 n_batches = n_batches + 1 output_fake = netD(x_fake.detach()) D_G_z = sigmoid(output_fake.mean()).item() fake_label = torch.ones_like(output_fake, device=device) errD_fake = loss(output_fake, fake_label) errD = errD_real + errD_fake errD.backward() optimizerD.step() # (2) Update G network if with_generator: netD.zero_grad() netG.zero_grad() output_fake = netD(x_fake) real_label = torch.zeros_like(output_fake, device=device) errG = loss(output_fake, real_label) if epoch >= num_epochs - matcher_epochs and just_train_classifier == False: errG = errG + m_score if just_train_classifier == False: errG.backward() optimizerG.step() ####################################################################### ####################################################################### sys.stdout.write("\r" + 'EPOCH [{}/{}] ..... {}-th batch: D_real = {:.3f} | D_fake = {:.3f}'.format(epoch+1, num_epochs, i+1, D_x, D_G_z)) #Progress with fixed noise if with_generator and just_train_classifier == False: with torch.no_grad(): x_fake_modified = netG(x_fake_to_modify) save_images(x_fake_to_modify[:3], x_fake_modified[:3], dataset, fake_material, epoch) tr_c_loss, tr_g_loss, tr_acc = eval_model((netD, netG), train_loader, device, epoch, num_epochs, matcher_epochs, with_generator = with_generator) train_history['train_c_loss'].append(tr_c_loss.item()) train_history['train_g_loss'].append(tr_g_loss.item()) train_history['train_acc'].append(tr_acc) val_c_loss, val_g_loss, val_acc = eval_model((netD, netG), valid_loader, device, epoch, num_epochs, matcher_epochs, with_generator = with_generator) train_history['val_c_loss'].append(val_c_loss.item()) train_history['val_g_loss'].append(val_g_loss.item()) train_history['val_acc'].append(val_acc) # display the training loss print() print( '\n>> Train: C_loss = {:.3f} |'.format(tr_c_loss.item()) + ' G_loss = {:.3f} |'.format(tr_g_loss.item()) + ' Acc = {:.3f}'.format(tr_acc) ) print( '\n>> Valid: C_loss = {:.3f} |'.format(val_c_loss.item()) + ' G_loss = {:.3f} |'.format(val_g_loss.item()) + ' Acc = {:.3f}'.format(val_acc) ) print() if epoch == num_epochs-1 and with_generator and just_train_classifier == False: score_matcher = score_matcher / n_batches print('\n>> Average matching score = {:.3f}'.format(score_matcher)) # save train/valid history plot_fn = output_path + 'LIVEGEN_history.png' plot_train_history(train_history, plot_fn=plot_fn) #load last model torch.save(netD.state_dict(), model_path + 'Discriminator.pth') if just_train_classifier == False: torch.save(netG.state_dict(), model_path + 'Generator.pth') return (netD, train_history)