def Predict(self, conll_path, BATCH_SIZE=5): nwords=0 nsents=0 with open(conll_path, 'r') as conllFP: for iSentence, sentence_batch in enumerate(read_conll_batch(conllFP, False, BATCH_SIZE),1): self.Init() # init initial stack and buffer pairs into sents sents = [self.init_sentence(s) for s in sentence_batch] hoffset = 1 if self.headFlag else 0 while sents: new_sents = [] exprs = [] for (stack, buf) in sents: if (len(buf)==1 and len(stack) == 0): continue new_sents.append((stack, buf)) routput, output = self.__evaluate(stack, buf, False) exprs.append((routput, output, stack, buf)) sents = new_sents # run forward on all the expressions _es = [_x[0] for _x in exprs]+[_x[1] for _x in exprs] if _es: dy.forward(_es) for routput, output, stack, buf in exprs: scores = self.exprs_to_scores(routput, output, stack, buf, False) best = max(chain(*scores), key = itemgetter(2) ) self.apply_action(best, stack, buf, hoffset) renew_cg() for sent in sentence_batch: yield sent
def train(self, trees, data_train=None): print_logger = PrintLogger() pool = Pool(self.options.concurrent_count) print_per = (100 // self.options.batch_size + 1) * self.options.batch_size self.sent_embeddings.rnn.set_dropout(self.options.lstm_dropout) for sentence_idx, batch_idx, batch_trees in split_to_batches( trees, self.options.batch_size): if sentence_idx % print_per == 0 and sentence_idx != 0: print_logger.print(sentence_idx) sessions = [self.training_session(tree, print_logger, pool) for tree in batch_trees] batch_size_2 = int(math.ceil(len(sessions) / 2) + 0.5) assert batch_size_2 * 2 >= len(sessions) for _, _, sub_sessions in split_to_batches( sessions, batch_size_2): exprs = [i for session in sub_sessions for i in next(session)] if exprs: dn.forward(exprs) futures = [next(session) for session in sub_sessions] loss = dn.esum([next(session) for session in sessions]) / len(sessions) # update print_logger.total_loss += loss.scalar_value() loss.backward() self.optimizer.update() dn.renew_cg()
def train_gen(self, sentences, update=True): print_logger = PrintLogger() pool = Pool(self.options.concurrent_count) self.network.sent_embedding.rnn.set_dropout(self.options.lstm_dropout) print_per = (100 // self.options.batch_size + 1) * self.options.batch_size for sentence_idx, batch_idx, batch_sentences in split_to_batches( sentences, self.options.batch_size): if sentence_idx % print_per == 0 and sentence_idx != 0: print_logger.print(sentence_idx) sessions = [self.training_session(sentence, print_logger, pool) for sentence in batch_sentences] all_exprs = [next(i) for i in sessions] if all_exprs: dn.forward(all_exprs) # spawn decoders for i in sessions: next(i) all_labels_exprs = [j for i in sessions for j in next(i)] if all_labels_exprs: dn.forward(all_labels_exprs) loss = sum(next(i) for i in sessions) / len(sessions) print_logger.total_loss_value += loss.value() if update: loss.backward() self.optimizer.update() dn.renew_cg() yield (loss if not update else None)
def predict(self, trees): self.sent_embeddings.rnn.disable_dropout() for sentence_idx, batch_idx, batch_trees in split_to_batches( trees, self.options.batch_size): sessions = [self.predict_session(tree) for tree in batch_trees] exprs = [i for session in sessions for i in next(session)] dn.forward(exprs) for session in sessions: yield next(session) dn.renew_cg()
def rerank(data_file, filename, scoring_model, qrels, qrels_file): queries_to_rerank = rerank_query_generator_ram(data_file) res_dict = {'questions': []} pred_batch_size = 100 for q in tqdm(queries_to_rerank, desc='queries', dynamic_ncols=True): query = q['token_inds'] query_idf = q['idf'] scores = [] dev_batches = chunks(range(len(q['retrieved_documents']['doc_list'])), pred_batch_size) for batch in dev_batches: batch_preds = [] dy.renew_cg() # new computation graph for i in batch: doc = q['retrieved_documents']['doc_list'][i] doc_bm25 = q['retrieved_documents']['doc_normBM25'][i] doc_overlap = q['retrieved_documents']['doc_overlap'][i] batch_preds.append( scoring_model.predict_doc_score(doc, query_idf, doc_bm25, doc_overlap)) dy.forward(batch_preds) scores += [pred.npvalue()[0] for pred in batch_preds] retr_scores = list(zip(q['retrieved_documents']['doc_ids'], scores)) sorted_retr_scores = sorted(retr_scores, key=lambda x: x[1], reverse=True) res_dict['questions'].append({ 'id': q['id'], 'documents': [ 'http://www.ncbi.nlm.nih.gov/pubmed/' + d[0] for d in sorted_retr_scores ] }) path_bioasq = write_bioasq_results_dict(res_dict, filename) trec_eval_metrics = trec_eval_custom(qrels_file, path_bioasq) for i in range(len(res_dict['questions'])): res_dict['questions'][i]['documents'] = res_dict['questions'][i][ 'documents'][:10] path_bioasq = write_bioasq_results_dict(res_dict, filename + '_top10') bioasq_metrics = bioasq_eval_custom(path_bioasq, qrels_file) precision_at_5 = get_precision_at_k(res_dict, qrels, 5) reported_results = { 'P_5': precision_at_5, 'MAP(bioasq)': bioasq_metrics['map'], 'GMAP(bioasq)': bioasq_metrics['gmap'] } return reported_results
def compose(self, composed_words, batch_size): outputs = [[] for _ in range(batch_size)] exprs = [] # Batching expression for expr_list, batch_num, position, start, end in composed_words: self.set_word(self.src_sent[batch_num][start:end]) expr = self.transduce(expr_list) if expr is not None: outputs[batch_num].append(expr) exprs.append(expr) dy.forward(exprs) return outputs
def predict_logprobs(self, X, Y): """ Returns the log probabilities of the predictions for this model (batched version). Returns a matrix of log probabilities. @param X: the input indexes from which to predict @param Y: a list of references indexes for which to extract the prob. @return the matrix of predicted logprobabilities for each of the provided ref y in Y as a numpy array """ assert (len(X) == len(Y)) assert (all([len(x) == len(y) for x, y in zip(X, Y)])) nlines = len(X) X = zip(*X) #transposes the batch Y = zip(*Y) #transposes the batch if self.tied: dy.renew_cg() state = self.rnn.initial_state() E = dy.parameter(self.embedding_matrix) preds = [] lookups = [dy.pick_batch(E, xcolumn) for xcolumn in X] outputs = state.transduce(lookups) ypred_batch = [ dy.pickneglogsoftmax_batch(E * lstm_out, y) for lstm_out, y in zip(outputs, Y) ] dy.forward(ypred_batch) if nlines > 1: preds = [(-col.npvalue()).tolist()[0] for col in ypred_batch] else: preds = [(-col.npvalue()).tolist() for col in ypred_batch] return list(zip(*preds)) #final back transposition else: dy.renew_cg() state = self.rnn.initial_state() O = dy.parameter(self.output_weights) E = dy.parameter(self.embedding_matrix) preds = [] lookups = [dy.pick_batch(E, xcolumn) for xcolumn in X] outputs = state.transduce(lookups) ypred_batch = [ dy.pickneglogsoftmax_batch(O * lstm_out, y) for lstm_out, y in zip(outputs, Y) ] dy.forward(ypred_batch) preds = [(-col.npvalue()).tolist()[0] for col in ypred_batch] if nlines > 1: preds = [(-col.npvalue()).tolist()[0] for col in ypred_batch] else: preds = [(-col.npvalue()).tolist() for col in ypred_batch] return list(zip(*preds)) #final back transposition
def calc_nll(self, src_batch, trg_batch) -> dy.Expression: self.actions.clear() self.outputs.clear() event_trigger.start_sent(src_batch) batch_loss = [] # For every item in the batch for src, trg in zip(src_batch, trg_batch): # Initial state with no read/write actions being taken current_state = self._initial_state(src) src_len = src.sent_len() # Reading + Writing src_encoding = [] loss_exprs = [] now_action = [] outputs = [] # Simultaneous greedy search while not self._stoping_criterions_met(current_state, trg): # Define action based on state action = self.next_action(current_state, src_len, len(src_encoding)) if action == self.Action.READ: # Reading + Encoding current_state = current_state.read(src) src_encoding.append(current_state.encoder_state.output()) else: # Predicting next word current_state = current_state.calc_context(src_encoding) current_output = self.add_input( current_state.prev_written_word, current_state) # Calculating losses ground_truth = self._select_ground_truth( current_state, trg) loss_exprs.append( self.decoder.calc_loss(current_output.state, ground_truth)) # Use word from ref/model depeding on settings next_word = self._select_next_word(ground_truth, current_output.state) # The produced words outputs.append(next_word) current_state = current_state.write(next_word) now_action.append(action.value) self.actions.append(now_action) self.outputs.append(outputs) # Accumulate loss batch_loss.append(dy.esum(loss_exprs)) dy.forward(batch_loss) loss = dy.esum(batch_loss) return loss if not self.freeze_decoder_param else dy.nobackprop(loss)
def predict(self, graphs): self.network.sent_embedding.rnn.disable_dropout() for sentence_idx, batch_idx, batch_sentences in split_to_batches( graphs, self.options.batch_size): sessions = [self.predict_session(sentence) for sentence in batch_sentences] all_exprs = [next(i) for i in sessions] if all_exprs: dn.forward(all_exprs) all_labels_exprs = [j for i in sessions for j in next(i)] if all_labels_exprs: dn.forward(all_labels_exprs) for i in sessions: yield next(i) dn.renew_cg()
def predict(self, trees, return_derivation=False): print_logger = PrintLogger() for sent_idx, batch_idx, batch_trees in split_to_batches( trees, self.options.batch_size): sessions = [self.training_session(tree, print_logger) for tree in batch_trees] exprs = [expr for session in sessions for expr in next(session)] dn.forward(exprs) for tree, session in zip(batch_trees, sessions): final_beam_item = next(session) graph = final_beam_item.sub_graph.graph if return_derivation: yield tree.extra["ID"], graph, list(self.construct_derivation(final_beam_item)) else: yield tree.extra["ID"], graph dn.renew_cg()
def train(self, trees): print_logger = PrintLogger() print_per = (100 // self.options.batch_size + 1) * self.options.batch_size for sent_idx, batch_idx, batch_trees in split_to_batches( trees, self.options.batch_size): if sent_idx % print_per == 0 and sent_idx != 0: print_logger.print(sent_idx) sessions = [self.training_session(tree, print_logger, self.derivations[tree.extra["ID"]]) for tree in batch_trees] exprs = [expr for session in sessions for expr in next(session)] dn.forward(exprs) loss = sum(next(session) for session in sessions) / len(sessions) print_logger.total_loss += loss.value() loss.backward() self.optimizer.update() dn.renew_cg()
def predict(self, sentences): self.network.sent_embedding.rnn.disable_dropout() pool = Pool(self.options.concurrent_count) for sentence_idx, batch_idx, batch_sentences in split_to_batches( sentences, self.options.test_batch_size): sessions = [self.predict_session(sentence, pool) for sentence in batch_sentences] all_exprs = [next(i) for i in sessions] if all_exprs: dn.forward(all_exprs) # spawn decoders for i in sessions: next(i) all_labels_exprs = [j for i in sessions for j in next(i)] if all_labels_exprs: dn.forward(all_labels_exprs) for i in sessions: yield next(i) dn.renew_cg()
def embed_sent(self, x: Any) -> expression_seqs.ExpressionSequence: """Embed a full sentence worth of words. By default, just do a for loop. Args: x: This will generally be a list of word IDs, but could also be a list of strings or some other format. It could also be batched, in which case it will be a (possibly masked) :class:`xnmt.batcher.Batch` object Returns: An expression sequence representing vectors of each word in the input. """ # single mode if not batchers.is_batched(x): expr = expression_seqs.ExpressionSequence( expr_list=[self.embed(word) for word in x]) # minibatch mode elif type(self) == LookupEmbedder: embeddings = [] for word_i in range(x.sent_len()): batch = batchers.mark_as_batch( [single_sent[word_i] for single_sent in x]) embeddings.append(self.embed(batch)) expr = expression_seqs.ExpressionSequence(expr_list=embeddings, mask=x.mask) else: assert type( x[0] ) == sent.SegmentedSentence, "Need to use CharFromWordTextReader for non standard embeddings." embeddings = [] all_embeddings = [] for sentence in x: embedding = [] for i in range(sentence.len_unpadded()): embed_word = self.embed(sentence.words[i]) embedding.append(embed_word) all_embeddings.append(embed_word) embeddings.append(embedding) # Useful when using dy.autobatch dy.forward(all_embeddings) all_embeddings.clear() # Pad the results expr = batchers.pad_embedding(embeddings) return expr
def evaluate(data, graph, vocab, outputFile): start = time.time() output = open(outputFile, 'w', encoding='utf-8') arc_total_test, arc_correct_test, rel_total_test, rel_correct_test = 0, 0, 0, 0 for onebatch in data_iter(data, config.test_batch_size, False, False): dy.renew_cg() batch_arc_probs, batch_rel_probs = [], [] for words, extwords, tags, heads, rels in sentences_numberize(onebatch, vocab): arc_probs, rel_probs = graph.parse(words, extwords, tags) batch_arc_probs.append(arc_probs) batch_rel_probs.append(rel_probs) dy.forward(batch_arc_probs + batch_rel_probs) batch_size = len(onebatch) for index in range(batch_size): seq_len = len(onebatch[index]) arc_probs = batch_arc_probs[index].npvalue() arc_probs = np.transpose(np.reshape(arc_probs, (seq_len, seq_len), 'F')) rel_probs = batch_rel_probs[index].npvalue() rel_probs = np.transpose(np.reshape(rel_probs, (vocab.rel_size, seq_len, seq_len), 'F')) arc_pred = arc_argmax(arc_probs, seq_len) rel_probs = rel_probs[np.arange(len(arc_pred)), arc_pred] rel_pred = rel_argmax(rel_probs, seq_len, vocab.ROOT) tree = append2Tree(arc_pred, rel_pred, vocab, onebatch[index]) printDepTree(output, tree) arc_total, arc_correct, rel_total, rel_correct = evalDepTree(tree, onebatch[index]) arc_total_test += arc_total arc_correct_test += arc_correct rel_total_test += rel_total rel_correct_test += rel_correct output.close() uas = arc_correct_test * 100.0 / arc_total_test las = rel_correct_test * 100.0 / rel_total_test end = time.time() print('sentence num:' + str(len(data)) + ', parse time: ', end - start) return arc_correct_test, rel_correct_test, arc_total_test, uas, las
def predict(self, trees): self.span_ebd_network.rnn.disable_dropout() pool = ThreadPool(self.options.concurrent_count) for sentence_idx, batch_idx, batch_trees in split_to_batches( trees, len(self.decoders)): self.span_ebd_network.init_special() sessions = [ self.predict_session(tree, pool, decoder) for tree, decoder in zip(batch_trees, self.decoders) ] # stage1: generate all expressions and forward expressions = [j for i in sessions for j in next(i)] dn.forward(expressions) # stage2: spawn all decoders for session in sessions: next(session) # stage3: get all results for session in sessions: yield next(session) dn.renew_cg()
def calc_nll(self, src_batch, trg_batch) -> losses.LossExpr: event_trigger.start_sent(src_batch) self.create_trajectories(src_batch, trg_batch, force_oracle=not self._is_action_forced()) batch_loss = [] for src, trg, decoder_state in zip(src_batch, trg_batch, self.decoder_states): seq_loss = [ self.decoder.calc_loss(decoder_state[i], trg[i]) for i in range(len(decoder_state)) ] batch_loss.append(dy.esum(seq_loss)) dy.forward(batch_loss) total_loss = dy.concatenate_to_batch(batch_loss) total_units = [ trg_batch[i].len_unpadded() for i in range(trg_batch.batch_size()) ] return losses.LossExpr(total_loss, total_units)
def calc_policy_nll(self, src_batch, trg_batch) -> losses.LossExpr: assert self.policy_network is not None event_trigger.start_sent(src_batch) self.create_trajectories(src_batch, trg_batch, force_oracle=not self._is_action_forced()) batch_loss = [] for src, action, model_states in zip(src_batch, self.actions, self.model_states): policy_actions = model_states[-1].find_backward("policy_action") seq_ll = [ dy.pick(act.log_likelihood, act.content) for act in policy_actions ] batch_loss.append(-dy.esum(seq_ll)) dy.forward(batch_loss) total_loss = dy.concatenate_to_batch(batch_loss) total_units = [len(x) for x in self.actions] return losses.LossExpr(total_loss, total_units)
def train_gen(self, graphs, update=True, extra=None): """ :type graphs: list[graph_utils.Graph] """ self.logger = PrintLogger() self.network.sent_embedding.rnn.set_dropout(self.options.lstm_dropout) print_per = (100 // self.options.batch_size + 1) * self.options.batch_size if extra is not None: for sentence_idx, batch_idx, batch_sentences in split_to_batches( extra, self.options.batch_size): if sentence_idx % print_per == 0 and sentence_idx != 0: self.logger.print(sentence_idx) sessions = [self.training_session(sentence, self.logger, loose_var=self.options.loose) for sentence in batch_sentences] all_exprs = [next(i) for i in sessions] if all_exprs: dn.forward(all_exprs) all_labels_exprs = [j for i in sessions for j in next(i)] if all_labels_exprs: dn.forward(all_labels_exprs) loss = sum(next(i) for i in sessions) / len(sessions) self.logger.total_loss_value += loss.value() if update: loss.backward() self.trainer.update() dn.renew_cg() sessions.clear() for sentence_idx, batch_idx, batch_sentences in split_to_batches( graphs, self.options.batch_size): if sentence_idx % print_per == 0 and sentence_idx != 0: self.logger.print(sentence_idx) sessions = [self.training_session(sentence, self.logger) for sentence in batch_sentences] all_exprs = [next(i) for i in sessions] if all_exprs: dn.forward(all_exprs) all_labels_exprs = [j for i in sessions for j in next(i)] if all_labels_exprs: dn.forward(all_labels_exprs) loss = sum(next(i) for i in sessions) / len(sessions) self.logger.total_loss_value += loss.value() if update: loss.backward() self.trainer.update() dn.renew_cg() sessions.clear() yield (loss if not update else None)
def compose(self, composed_words, sample_size, batch_size): batches = [] batch_maps = [] batch_words = [] seq_len = np.zeros((sample_size, batch_size), dtype=int) composed_words = sorted(composed_words, key=lambda x: x[5] - x[4]) # Batching expression now_length = -1 for expr_list, sample_num, batch_num, position, start, end in composed_words: length = end - start if length != now_length: now_length = length now_map = {} now_batch = [] now_words = [] now_idx = 0 batches.append(now_batch) batch_maps.append(now_map) batch_words.append(now_words) now_batch.append(expr_list) now_words.append(self.src_sent[batch_num][start:end]) now_map[now_idx] = (sample_num, batch_num, position) seq_len[sample_num, batch_num] += 1 now_idx += 1 # Composing outputs = [[[None for _ in range(seq_len[i, j])] for j in range(batch_size)] for i in range(sample_size)] expr_list = [] for batch, batch_map, batch_word in zip(batches, batch_maps, batch_words): self.set_words(batch_word) results = self.transduce(dy.concatenate_to_batch(batch)) results.value() for idx, (sample_num, batch_num, position) in batch_map.items(): expr_list.append(dy.pick_batch_elem(results, idx)) outputs[sample_num][batch_num][position] = expr_list[-1] dy.forward(expr_list) return outputs
def predict_logprobs(self, X, Y, hidden_out=False): """ Returns the log probabilities of the predictions for this model (batched version). @param X: the input indexes from which to predict (each xdatum is expected to be an iterable of integers) @param Y: a list of references indexes for which to extract the prob. @param hidden_out: outputs an additional list of hidden dimension vectors @return the list of predicted logprobabilities for each of the provided ref y in Y """ #TODO: implement the hidden_out output assert (len(X) == len(Y)) assert (all(len(x) == self.input_length for x in X)) preds = [] if self.tied: dy.renew_cg() W = dy.parameter(self.hidden_weights) E = dy.parameter(self.embedding_matrix) for x, y in zip(X, Y): embeddings = [dy.pick(E, widx) for widx in x] xdense = dy.concatenate(embeddings) ypred = dy.pickneglogsoftmax(E * dy.tanh(W * xdense), y) preds.append(ypred) dy.forward(preds) return [-ypred.value() for ypred in preds] else: dy.renew_cg() O = dy.parameter(self.output_weights) W = dy.parameter(self.hidden_weights) E = dy.parameter(self.embedding_matrix) for x, y in zip(X, Y): embeddings = [dy.pick(E, widx) for widx in x] xdense = dy.concatenate(embeddings) ypred = dy.pickneglogsoftmax(O * dy.tanh(W * xdense), y) preds.append(ypred) dy.forward(preds) return [-ypred.value() for ypred in preds]
def train(self, trees, data_train=None): print_logger = PrintLogger() print_per = (100 // self.options.batch_size + 1) * self.options.batch_size self.sent_embeddings.rnn.set_dropout(self.options.lstm_dropout) for sentence_idx, batch_idx, batch_trees in split_to_batches( trees, self.options.batch_size): if sentence_idx % print_per == 0 and sentence_idx != 0: print_logger.print(sentence_idx) sessions = [ self.training_session(tree, print_logger) for tree in batch_trees ] exprs = [i for session in sessions for i in next(session)] dn.forward(exprs) loss = dn.esum([next(session) for session in sessions]) / len(sessions) # update print_logger.total_loss += loss.scalar_value() loss.backward() self.optimizer.update() dn.renew_cg()
def train(self, sentences): self.span_ebd_network.rnn.set_dropout(self.options.lstm_dropout) self.span_ebd_network.init_special() pool = ThreadPool(self.options.concurrent_count) print_logger = PrintLogger() print_per = (100 // self.options.batch_size + 1) * self.options.batch_size for sentence_idx, batch_idx, batch_trees in split_to_batches( sentences, self.options.batch_size): if sentence_idx != 0 and sentence_idx % print_per == 0: print_logger.print(sentence_idx) self.span_ebd_network.init_special() sessions = [ self.train_session(tree, print_logger, pool, decoder) for tree, decoder in zip(batch_trees, self.decoders) ] batch_size_2 = int(math.ceil(len(sessions) / 2) + 0.5) assert batch_size_2 * 2 >= len(sessions) for _, _, sub_sessions in split_to_batches(sessions, batch_size_2): # stage1: generate all expressions and forward expressions = [j for i in sub_sessions for j in next(i)] dn.forward(expressions) # stage2: spawn all decoders for session in sub_sessions: next(session) # stage3: get all losses loss = dn.esum([next(session) for session in sessions]) loss /= len(sessions) print_logger.total_loss += loss.value() loss.backward() self.optimizer.update() dn.renew_cg() self.span_ebd_network.init_special()
confusion = [[0 for _ in xrange(10)] for _ in xrange(10)] correct = 0 dev_start = time.time() for s in range(0, len(testing), args.minibatch_size): dy.renew_cg() classify.renew_cg() e = min(len(testing), s + args.minibatch_size) minibatch = testing[s:e] scores = [] for lbl, img in minibatch: x = dy.inputVector(img) logits = classify(x) scores.append((lbl, logits)) # This evaluates all the logits in a batch if autobatching is on. dy.forward([logits for _, logits in scores]) # now we can retrieve the batch-computed logits cheaply for lbl, logits in scores: prediction = np.argmax(logits.npvalue()) if lbl == prediction: correct += 1 confusion[prediction][lbl] += 1 dev_end = time.time() acc = float(correct) / len(testing) dev_time += dev_end - dev_start print("Held out accuracy {} ({} instances/sec)".format( acc, len(testing) / (dev_end - dev_start))) print ' ' + ''.join( ('T' + str(x)).ljust(6) for x in xrange(10))
print 'epoch ' + str( epoch) + ' batch ' + str(i_batch) + ' loss ' + str( batch_loss.npvalue()) # this calls forward on the batch batch_loss.backward() trainer.update() training_duration = datetime.datetime.now() - pretrain_time print 'Done! training took: ' + str(training_duration) + '\n' print "\n\nPrediction time!\n" # prediction code: correct = 0 for batch in test_batches: dy.renew_cg() # new computation graph batch_preds = [] for sequence, label in batch: vecs = [embeds[char2int[i]] for i in sequence] preds = dy.softmax(acceptor(vecs)) batch_preds.append(preds) # now that we accumulated the prediction expressions, # we run forward on all of them: dy.forward(batch_preds) # and now we can efficiently access the individual values: for preds in batch_preds: vals = preds.npvalue() if np.argmax(vals) == label: correct += 1 print 'accuracy: ' + str(float(correct) / len(test))
def evaluate(file, char_acceptor, word_acceptor, char_embed, word_embed, crf_acceptor=None): '''evaluate performance of model on file @param file: string, path to test/dev file @param char_acceptor: CharBiLSTMAcceptor @param word_acceptor: WordBiLSTMAcceptor @param char_embed: lookup parameter @param word_embed: lookup parameter @param crf_acceptor: CRFAcceptor, default=None @return: float, float: accuracy, f1 score ''' # extract evaluating samples eval_samples = list(readSample(file)) # counts are used to calculate acc, f1 count_tag = 0 count_correct_tag = 0 count_gold_chunk = 0 count_pred_chunk = 0 count_correct_chunk = 0 num_batch = len(list(genBatch(eval_samples))) showpro = ShowProcess(num_batch, "evaluating done.") for batch in genBatch(eval_samples): # for each batch dy.renew_cg() logitss = [] # shape=(#sentence,#word), elem_type=expression goldss = [] # gold tags, shape=(#sentence,#word), elem_type=int for wids, tids, cidss in batch: # for each sentence # feed into biLSTMs and get logits wembeds1 = [word_embed[wid] for wid in wids] wembeds2 = [ char_acceptor([char_embed[cid] for cid in cids]) for cids in cidss ] wembeds = [ dy.concatenate([embed1, embed2]) for embed1, embed2 in zip(wembeds1, wembeds2) ] logitss.append(word_acceptor(wembeds)) # record gold tags goldss.append(tids) # in order to use dy.forward, flatten logitss into a list of expressions logits_flt = [] for logits in logitss: logits_flt.extend(logits) # do farward # note: no backward or update here in evaluating stage dy.forward(logits_flt) # use logits to predict # predss: shape=(#sentence,#word), elem_type=int if crf_acceptor is None: # do not use crf predss = [[np.argmax(vec.npvalue()) for vec in vecs] \ for vecs in logitss] else: # use crf predss = [crf_acceptor.predict(vecs) for vecs in logitss] # update counts for golds, preds in zip(goldss, predss): # for each sentence gold_chunks = get_chunks(golds) pred_chunks = get_chunks(preds) count_gold_chunk += len(gold_chunks) count_pred_chunk += len(pred_chunks) count_correct_chunk += len(gold_chunks & pred_chunks) count_tag += len(golds) golds_np = np.asarray(golds, dtype=np.int8) preds_np = np.asarray(preds, dtype=np.int8) correct = golds_np == preds_np count_correct_tag += np.sum(correct) showpro() # calculate accuracy acc = float(count_correct_tag) / count_tag # calculate f1 score p = float(count_correct_chunk) / count_pred_chunk r = float(count_correct_chunk) / count_gold_chunk f1 = 2 * p * r / (p + r) if p + r > 0 else 0.0 return acc, f1
def train(self, trees): if isinstance(self.scorer_network, CountBasedHRGScorer): logger.info("No need to train a count-based scorer.") return total_count = sys.float_info.epsilon correct_count = 0 pending = [] total_loss = 0 for tree_idx, tree in enumerate(trees): # if tree_idx == 5000: # return if (tree_idx + 1) % 100 == 0: logger.info( "Sent {}, Correctness: {:.2f}, loss: {:.2f}".format( tree_idx + 1, correct_count / total_count * 100, total_loss)) total_count = sys.float_info.epsilon correct_count = 0 total_loss = 0 sent_id = tree.extra["ID"] derivations = self.derivations[sent_id] # type: List[CFGRule] sentence_interface = tree.to_sentence() self.span_ebd_network.init_special() span_features = self.span_ebd_network.get_span_features( sentence_interface) cfg_nodes = list(tree.generate_rules()) # type: List[ConstTree] assert len(derivations) == len(cfg_nodes) for gold_rule, tree_node in zip(derivations, cfg_nodes): if tree_node.tag.endswith("#0"): continue try: correspondents = set( self.rule_lookup(tree_node, True).items()) except ValueError as e: print(e) continue # print(span_features[tree_node.span].npvalue().shape) pending.append((self.scorer_network.get_best_rule( span_features[tree_node.span], correspondents, gold_rule), gold_rule)) if tree_idx % self.options.batch_size == 0 or tree_idx == len( trees) - 1: # generate expressions exprs = [] for item, gold_rule in pending: exprs.extend(next(item)) # do batch calculation if exprs: dn.forward(exprs) # calculate loss loss = dn.scalarInput(0.0) for item, gold_rule in pending: best_rule, this_loss, real_best_rule = next(item) if this_loss is not None: total_count += 1 loss += this_loss if real_best_rule == gold_rule: correct_count += 1 loss.forward() total_loss += loss.scalar_value() loss.backward() self.optimizer.update() dn.renew_cg() pending = []
if (i > 0) and (i % dev_report == 0): confusion = [[0 for _ in range(10)] for _ in range(10)] correct = 0 dev_start = time.time() for s in range(0, len(testing), args.minibatch_size): dy.renew_cg() e = min(len(testing), s + args.minibatch_size) minibatch = testing[s:e] scores = [] for lbl, img in minibatch: x = dy.inputVector(img) logits = classify(x) scores.append((lbl, logits)) # This evaluates all the logits in a batch if autobatching is on. dy.forward([logits for _, logits in scores]) # now we can retrieve the batch-computed logits cheaply for lbl, logits in scores: prediction = np.argmax(logits.npvalue()) if lbl == prediction: correct += 1 confusion[prediction][lbl] += 1 dev_end = time.time() acc = float(correct) / len(testing) dev_time += dev_end - dev_start print(("Held out accuracy {} ({} instances/sec)".format( acc, len(testing) / (dev_end - dev_start)))) print(' ' + ''.join(('T'+str(x)).ljust(6) for x in range(10))) for p, row in enumerate(confusion): s = 'P' + str(p) + ' '