def dev(model): data_helper = DataHelper(mode='dev') total_pred = 0 correct = 0 accuracy = 0 b_size = len(data_helper.label) print('*' * 100) print('dev set total:', b_size) loss_func = torch.nn.MSELoss(reduction='sum') ##reduction='sum' loss_mae = torch.nn.L1Loss(reduction='sum') iter = 0 total_loss = 0 for content, label, _ in data_helper.batch_iter(batch_size=b_size, num_epoch=1): iter += 1 model.eval() ##need modify the regression task will minimize the mse error pred = model(content) pred_sq = torch.squeeze(pred, 1) loss = loss_func(pred_sq.cpu().data, label.cpu()) #------------------------------------------------# error = loss_mae(pred_sq.cpu().data, label.cpu()) # error = mean_absolute_error(pred_sq.cpu().data, label.cpu()) accuracy += error total_pred = len(label) total_pred = float(total_pred) accuracy = float(accuracy) #return the overall accuracy return (accuracy / total_pred), (float(loss) / total_pred)
def dev(model, dataset): data_helper = DataHelper(dataset, mode='dev') total_pred = 0 correct = 0 iter = 0 for content, label, _ in data_helper.batch_iter(batch_size=64, num_epoch=1): iter += 1 model.eval() logits = model(content) pred = torch.argmax(logits, dim=1) correct_pred = torch.sum(pred == label) correct += correct_pred total_pred += len(content) total_pred = float(total_pred) correct = correct.float() # print(torch.div(correct, total_pred)) return torch.div(correct, total_pred)
def test(model): model_name = 'temp_model' model = torch.load(os.path.join('.', model_name + '.pkl')) data_helper = DataHelper(mode='test') b_size = len(data_helper.label) print('test set total:', b_size) loss_func = torch.nn.MSELoss(reduction='sum') total_pred = 0 correct = 0 iter = 0 accuracy = 0 pre_score = [] score = [] for content, label, _ in data_helper.batch_iter(batch_size=b_size, num_epoch=1): iter += 1 model.eval() pred = model(content) pred_sq = torch.squeeze(pred, 1) loss = loss_func(pred_sq, label.float()) error = mean_absolute_error(pred_sq.cpu().data, label.cpu(), multioutput='raw_values') accuracy += error total_pred = len(label) pre_score.append(pred_sq.cpu().data) score.append(label.cpu()) total_pred = float(total_pred) accuracy = float(accuracy) print(torch.div(accuracy, total_pred)) _ = result_pull(pre_score, score) # print('iter is:', iter) print('pred result:%.2f, true score:%d' % (pre_score, score)) return torch.div(accuracy, total_pred)
def test(model_name, dataset): model = torch.load(os.path.join('.', model_name + '.pkl')) data_helper = DataHelper(dataset, mode='test') total_pred = 0 correct = 0 iter = 0 for content, label, _ in data_helper.batch_iter(batch_size=64, num_epoch=1): iter += 1 model.eval() logits = model(content) pred = torch.argmax(logits, dim=1) correct_pred = torch.sum(pred == label) correct += correct_pred total_pred += len(content) total_pred = float(total_pred) correct = correct.float() # print(torch.div(correct, total_pred)) return torch.div(correct, total_pred).to('cpu')
def train(ngram, name, bar, drop_out, dataset, is_cuda=False, edges=True): print('load data helper.') data_helper = DataHelper(dataset, mode='train') if os.path.exists(os.path.join('.', name+'.pkl')) and name != 'temp_model': print('load model from file.') model = torch.load(os.path.join('.', name+'.pkl')) else: print('new model.') if name == 'temp_model': name = 'temp_model_%s' % dataset # edges_num, edges_matrix = edges_mapping(len(data_helper.vocab), data_helper.content, ngram) edges_weights, edges_mappings, count = cal_PMI(dataset=dataset) model = Model(class_num=len(data_helper.labels_str), hidden_size_node=200, vocab=data_helper.vocab, n_gram=ngram, drop_out=drop_out, edges_matrix=edges_mappings, edges_num=count, trainable_edges=edges, pmi=edges_weights, cuda=is_cuda) print(model) if is_cuda: print('cuda') model.cuda() loss_func = torch.nn.CrossEntropyLoss() optim = torch.optim.Adam(model.parameters(), weight_decay=1e-6) iter = 0 if bar: pbar = tqdm.tqdm(total=NUM_ITER_EVAL) best_acc = 0.0 last_best_epoch = 0 start_time = time.time() total_loss = 0.0 total_correct = 0 total = 0 for content, label, epoch in data_helper.batch_iter(batch_size=32, num_epoch=200): improved = '' model.train() logits = model(content) loss = loss_func(logits, label) pred = torch.argmax(logits, dim=1) correct = torch.sum(pred == label) total_correct += correct total += len(label) total_loss += loss.item() optim.zero_grad() loss.backward() optim.step() iter += 1 if bar: pbar.update() if iter % NUM_ITER_EVAL == 0: if bar: pbar.close() val_acc = dev(model, dataset=dataset) if val_acc > best_acc: best_acc = val_acc last_best_epoch = epoch improved = '*' torch.save(model, name + '.pkl') if epoch - last_best_epoch >= EARLY_STOP_EPOCH: return name msg = 'Epoch: {0:>6} Iter: {1:>6}, Train Loss: {5:>7.2}, Train Acc: {6:>7.2%}' \ + 'Val Acc: {2:>7.2%}, Time: {3}{4}' \ # + ' Time: {5} {6}' print(msg.format(epoch, iter, val_acc, get_time_dif(start_time), improved, total_loss/ NUM_ITER_EVAL, float(total_correct) / float(total))) total_loss = 0.0 total_correct = 0 total = 0 if bar: pbar = tqdm.tqdm(total=NUM_ITER_EVAL) return name
Model.input_y: y_batch, Model.dropout_keep_prob: 1.0, Model.b_size: len(y_batch) } step, loss, accuracy = sess.run( [global_step, Model.cost, Model.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() print("DEV {}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) summary_op_out = sess.run(train_summary_op, feed_dict=feed_dict) train_summary_writer.add_summary(summary_op_out, step) return accuracy # Generate batches batches = datahelper.batch_iter(list(zip(train_set[0], train_set[1])), FLAGS.batch_size, FLAGS.num_epochs) ptr = 0 max_validation_acc = 0.0 for nn in range(sum_no_of_batches * FLAGS.num_epochs): batch = batches.__next__() if len(batch) < 1: continue x1_batch, y_batch = zip(*batch) if len(y_batch) < 1: continue train_step(x1_batch, y_batch) current_step = tf.train.global_step(sess, global_step) sum_acc = 0.0 if current_step % FLAGS.evaluate_every == 0: print("\nEvaluation:")
def train(ngram, name, bar, drop_out, dataset, is_cuda=False, edges=False): plotter = LossAccPlotter(title="This is an example plot", save_to_filepath="/tmp/my_plot.png", show_regressions=True, show_averages=True, show_loss_plot=True, show_acc_plot=True, show_plot_window=False, x_label="Epoch") print('load data helper.') data_helper = DataHelper(mode='train') b_size = len(data_helper.label) print('*' * 100) print('train set total:', b_size) if os.path.exists(os.path.join('.', name + '.pkl')) and name != 'temp_model': print('load model from file') model = torch.load(os.path.join('.', name + '.pkl')) else: print('new model') if name == 'temp_model': name == 'temp_model' edges_weights, edges_mappings, count = cal_PMI() ## -----------------************************** import the datahelper class to get the vocab-5 doc*****************------------------------ ## class_num = len(data_helper.labels_str) is changed, consider just a score model = Model(class_num=data_helper.labels_str, hidden_size_node=200, vocab=data_helper.vocab, n_gram=ngram, drop_out=drop_out, edges_matrix=edges_mappings, edges_num=count, trainable_edges=edges, pmi=edges_weights, cuda=is_cuda) ### --------------------------------------- ### print(model) if is_cuda: print('cuda') model.cuda() loss_func = torch.nn.MSELoss() loss_mae = torch.nn.L1Loss(reduction='sum') optim = torch.optim.Adam(model.parameters(), weight_decay=1e-3) iter = 0 if bar: pbar = tqdm.tqdm(total=NUM_ITER_EVAL) best_acc = 0.0 last_best_epoch = 0 start_time = time.time() total_loss = 0.0 total_correct = 0 total = 0 accuracy = 0.0 num_epoch = 500 weight_decays = 1e-4 for content, label, epoch in data_helper.batch_iter( batch_size=32, num_epoch=num_epoch): improved = '' model.train() pred = model(content) pred_sq = torch.squeeze(pred, 1) l2_reg = 0.5 * weight_decays * ( model.seq_edge_w.weight.to('cpu').detach().numpy()**2).sum() loss = loss_func(pred_sq, label.float()) + l2_reg #-------------------------------------------# error = loss_mae(pred_sq.cpu().data, label.cpu()) accuracy += error total += len(pred) ##batch size = len(label) total_loss += (loss.item() * len(pred)) total_correct += loss.item() optim.zero_grad() loss.backward() optim.step() iter += 1 if bar: pbar.update() if iter % NUM_ITER_EVAL == 0: if bar: pbar.close() val_acc, val_loss = dev(model) if val_acc < best_acc: best_acc = val_acc last_best_epoch = epoch improved = '* ' torch.save(model, name + '.pkl') msg = 'Epoch: {0:>6} Iter: {1:>6}, Train Loss: {5:>7.2}, Train Error: {6:>7.2}' \ + 'Val Acc: {2:>7.2}, Time: {3}{4}, val error:{7:>7.2}' \ # + ' Time: {5} {6}' print( msg.format(epoch, iter, val_acc, get_time_dif(start_time), improved, total_correct / (NUM_ITER_EVAL), float(accuracy) / float(total), val_loss)) plotter.add_values(epoch, loss_train=total_correct / (NUM_ITER_EVAL), acc_train=float(accuracy) / float(total), loss_val=val_loss, acc_val=best_acc) total_loss = 0.0 total_correct = 0 accuracy = 0.0 total = 0 if bar: pbar = tqdm.tqdm(total=NUM_ITER_EVAL) plotter.block() return name