def evaluate_corpus(trees, fm, network): accuracy = FScore() for tree in trees: predicted = Parser.parse(tree.sentence, fm, network) local_accuracy = predicted.compare(tree) accuracy += local_accuracy return accuracy
def compare(self, gold, advp_prt=True): """ returns (Precision, Recall, F-measure) """ predbracks = self.brackets(advp_prt) goldbracks = gold.brackets(advp_prt) correct = 0 for gb in goldbracks: if gb in predbracks: correct += min(goldbracks[gb], predbracks[gb]) pred_total = sum(predbracks.values()) gold_total = sum(goldbracks.values()) return FScore(correct, pred_total, gold_total)
def write_predicted(fname, trees, fm, network): """ Input trees being used only to carry sentences. """ f = open(fname, 'w') accuracy = FScore() for tree in trees: predicted = Parser.parse(tree.sentence, fm, network) local_accuracy = predicted.compare(tree) accuracy += local_accuracy topped = PhraseTree( symbol='TOP', children=[predicted], sentence=predicted.sentence, ) f.write(str(topped)) f.write('\n') f.close() return accuracy
def train(fm, args): train_data_file = args.train dev_data_file = args.dev epochs = args.epochs batch_size = args.batch_size unk_param = args.unk_param alpha = args.alpha beta = args.beta model_save_file = args.model print("this is train mode") start_time = time.time() network = Network(fm, args) optimizer = optimize.Adadelta(network.parameters(), eps=1e-7, rho=0.99) if GlobalNames.use_gpu: network.cuda() training_data = fm.gold_data_from_file(train_data_file) num_batches = -(-len(training_data) // batch_size) print('Loaded {} training sentences ({} batches of size {})!'.format( len(training_data), num_batches, batch_size, )) parse_every = -(-num_batches // 4) dev_trees = PhraseTree.load_trees(dev_data_file) print('Loaded {} validation trees!'.format(len(dev_trees))) best_acc = FScore() for epoch in xrange(1, epochs + 1): print('........... epoch {} ...........'.format(epoch)) total_cost = 0.0 total_states = 0 training_acc = FScore() np.random.shuffle(training_data) for b in xrange(num_batches): network.zero_grad() batch = training_data[(b * batch_size): ((b + 1) * batch_size)] batch_loss = None for example in batch: example_Loss, example_states, acc = Parser.exploration(example, fm, network, alpha, beta, unk_param) total_states += example_states if batch_loss is not None: batch_loss += example_Loss else: batch_loss = example_Loss training_acc += acc if GlobalNames.use_gpu: total_cost += batch_loss.cpu().data.numpy()[0] else: total_cost += batch_loss.data.numpy()[0] batch_loss.backward() optimizer.step() mean_cost = total_cost / total_states print( '\rBatch {} Mean Cost {:.4f} [Train: {}]'.format( b, mean_cost, training_acc, ), end='', ) sys.stdout.flush() if ((b + 1) % parse_every) == 0 or b == (num_batches - 1): dev_acc = Parser.evaluate_corpus( dev_trees, fm, network, ) print(' [Dev: {}]'.format(dev_acc)) if dev_acc > best_acc: best_acc = dev_acc torch.save(network, model_save_file) print(' [saved model: {}]'.format(model_save_file)) current_time = time.time() runmins = (current_time - start_time) / 60. print(' Elapsed time: {:.2f}m'.format(runmins))
def train(fm, args): train_data_file = args.train dev_data_file = args.dev epochs = args.epochs batch_size = args.batch_size unk_param = args.unk_param alpha = args.alpha beta = args.beta model_save_file = args.model print("this is train mode") start_time = time.time() network = SpanParserNN(fm, args) optimizer = optimize.Adadelta(network.parameters(), eps=1e-7, rho=0.99) # network.cuda() training_data = fm.gold_data_from_file(train_data_file) num_batches = -(-len(training_data) // batch_size) print('Loaded {} training sentences ({} batches of size {})!'.format( len(training_data), num_batches, batch_size, )) parse_every = -(-num_batches // 4) dev_trees = PhraseTree.load_trees(dev_data_file) print('Loaded {} validation trees!'.format(len(dev_trees))) best_acc = FScore() for epoch in xrange(1, epochs + 1): print('........... epoch {} ...........'.format(epoch)) total_cost = 0.0 total_states = 0 training_acc = FScore() np.random.shuffle(training_data) for b in xrange(num_batches): batch = training_data[(b * batch_size):((b + 1) * batch_size)] explore = [ Parser.exploration( example, fm, network, alpha=alpha, beta=beta, ) for example in batch ] for (_, acc) in explore: training_acc += acc batch = [example for (example, _) in explore] sum_loss = np.zeros(1) for example in batch: ## random UNKing ## for (i, w) in enumerate(example['w']): if w <= 2: continue freq = fm.word_freq_list[w] drop_prob = unk_param / (unk_param + freq) r = np.random.random() if r < drop_prob: example['w'][i] = 0 fwd, back = network.evaluate_word( example['w'], example['t'], ) for (left, right), correct in example['struct_data'].items(): scores = network(fwd, back, left, right, 'struct') probs = F.softmax(scores, dim=0) loss = -torch.log(probs[correct]) sum_loss += loss.data.numpy() loss.backward(retain_graph=True) total_states += len(example['struct_data']) for (left, right), correct in example['label_data'].items(): scores = network(fwd, back, left, right, 'label') probs = F.softmax(scores, dim=0) loss = -torch.log(probs[correct]) sum_loss += loss.data.numpy() loss.backward(retain_graph=True) total_states += len(example['label_data']) total_cost += sum_loss optimizer.step() network.zero_grad() mean_cost = total_cost / total_states print( '\rBatch {} Mean Cost {:.4f} [Train: {}]'.format( b, mean_cost, training_acc, ), end='', ) sys.stdout.flush() if ((b + 1) % parse_every) == 0 or b == (num_batches - 1): dev_acc = Parser.evaluate_corpus( dev_trees, fm, network, ) print(' [Val: {}]'.format(dev_acc)) if dev_acc > best_acc: best_acc = dev_acc torch.save(network, model_save_file) print(' [saved model: {}]'.format(model_save_file)) current_time = time.time() runmins = (current_time - start_time) / 60. print(' Elapsed time: {:.2f}m'.format(runmins))