def process_one_article(self, original_article_sents, original_abstract_sents, \ original_selected_ids, output_ids, oovs, \ attn_dists, p_gens, log_probs, counter): # Remove the [STOP] token from decoded_words, if necessary decoded_words = data.outputids2words(output_ids, self._vocab, oovs) try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string decoded_sents = data.words2sents(decoded_words) if FLAGS.single_pass: verbose = False if FLAGS.mode == 'eval' else True self.write_for_rouge(original_abstract_sents, decoded_sents, counter, verbose) # write ref summary and decoded summary to file, to eval with pyrouge later if FLAGS.decode_method == 'beam' and FLAGS.save_vis: original_article = ' '.join(original_article_sents) original_abstract = ' '.join(original_abstract_sents) article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, oovs) self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, \ attn_dists, p_gens, log_probs, counter, verbose) if FLAGS.save_pkl: self.save_result(original_article_sents, original_abstract_sents, \ original_selected_ids, decoded_sents, counter, verbose)
def decodeOneSample(self, batches): batch = batches[0] original_article = batch.original_articles[0] original_abstract = batch.original_abstracts[0] original_abstract_sents = batch.original_abstracts_sents[0] article_withunks = data.show_art_oovs(original_article, self._vocab) abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, batch.art_oovs[0]) best_hypothesis = beam_search.run_beam_search(self._session, self._model, self._vocab, batch, self._hps) output_ids = [int(t) for t in best_hypothesis.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, batch.art_oovs[0]) try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string self.write_for_rouge(original_abstract_sents, decoded_words, 0, original_article) self.rouge_eval() print_results(article_withunks, abstract_withunks, decoded_output) self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hypothesis.attn_dists, best_hypothesis.p_gens)
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 while True: batch = self._batcher.next_batch() # 1 example repeated across batch # if counter == 10: # batch = None if batch is None: # finished decoding dataset in single_pass mode d = [all_attn_dists,all_pgens,dec_words] output_fname = os.path.join(self._decode_dir, 'attn_dist_p_gens_data.pkl') pickle.dump(d,open(output_fname,'wb')) assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") tf.logging.info("Output has been saved in %s and %s. Now starting eval...", self._rouge_ref_dir, self._rouge_dec_dir) #Metrics here self.get_metrics(self._rouge_ref_dir,self._rouge_dec_dir) return original_article = batch.original_articles[0] # string original_query = batch.original_queries[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string #query_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: self.write_for_rouge(original_abstract_sents, decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded all_attn_dists.append(best_hyp.attn_dists) all_pgens.append(best_hyp.p_gens) dec_words.append(decoded_words) # self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens,count=counter) else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1-t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1-t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self): decode_relations = [] original_relations = [] while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode # todo after finish final batch, compute precision, recall and f1_score p, r, f1 = calculate_measure_cmp(decode_relations, original_relations) tf.logging.info("p: %.4f, r: %.4f, f1: %.4f", p, r, f1) assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words_webnlg( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string decode_relations.append(decoded_output) original_relations.append(original_abstract) print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 step = 0 while True: step += 1 batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert self._model.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") tf.logging.info("Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return t_start_decode = time.time() original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if self._model.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = self._model.model_decode(batch, self._sess, self._vocab) print(best_hyp.tokens) #best_hyp = None # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if self._model.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if self._model.single_pass: self.write_for_rouge(original_abstract_sents, decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1-t0 > SECS_UNTIL_NEW_CKPT: print('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1-t0) _ = util.load_checkpoint(self._saver, self._sess) t0 = time.time() t_end_decode = time.time() print('decode {}-th batch requires {} seconds'.format(step, int(t_end_decode - t_start_decode)))
def decode(self, ckpt_file=None): FLAGS = self._FLAGS # load latest checkpoint misc_utils.load_ckpt(self._saver, self._sess, self._ckpt_dir, ckpt_file) counter = 0 f = open(self._decode_dir, "w") while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode tf.logging.info( "Decoder has finished reading dataset for single_pass.") break original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch, FLAGS) # Extract the output ids from the hypothesis and convert back to # words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: # index of the (first) [STOP] symbol fst_stop_idx = decoded_words.index(data.STOP_DECODING) decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words # write ref summary and decoded summary to file, to eval with # pyrouge later # self.write_for_rouge(original_abstract_sents, decoded_words, counter) processed = self.depreciated_processing(decoded_words) f.write(processed + "\n") counter += 1 if counter % 100 == 0: print("%d sentences decoded" % counter) f.close()
def pair_wise_decode(self): f = os.path.join(FLAGS.data_path, "output.txt") outputfile = codecs.open(f, "w", "utf8") output_result = [] list_of_reference = [] while True: batch = self._batcher.next_pairwised_decode_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode logging.info("eval_finished") outputfile.close() break print(self._batcher.c_index) original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis result = self.eval_one_batch(self._sess, self._model, self._vocab, batch) # result = self.eval_one_batch(self._sess, self._model, self._vocab, batch) for i, instance in enumerate(result): if i == len(batch.art_oovs): break if i >= batch.real_length: print("eval done with {} isntances".format( len(output_result))) break out_words = data.outputids2words(instance, self._model._vocab_out, batch.art_oovs[i]) if data.STOP_DECODING in out_words: out_words = out_words[:out_words.index(data.STOP_DECODING)] output_now = " ".join(out_words) output_result.append(output_now) # refer = " ".join(refer) refer = batch.original_abstracts[i].strip() list_of_reference.append([refer]) outputfile.write(batch.original_articles[i] + '\t' + output_now + '\n') bleu = matrix.bleu_score(list_of_reference, output_result) acc = matrix.compute_acc(list_of_reference, output_result) print("bleu : {} acc : {}".format(bleu, acc)) return
def _decode(self): """ """ t0 = time.time() counter = 0 while True: batch = self.batcher._next_batch() if batch is None: assert FLAGS.onetime, "Dataset exhausted, but we are not in onetime mode" print('INFO: Decoder has finished reading dataset for onetime.') print('INFO: Output has been saved in {} and {}, start ROUGE eval...'.format(self.rouge_ref_dir, self.rouge_dec_dir)) results_dict = rouge_eval(self.rouge_ref_dir, self.rouge_dec_dir) rouge_log(results_dict, self.decode_dir) return original_article = batch.original_articles[0] original_abstract = batch.original_abstracts[0] article_withunks = data.show_art_oovs(original_article, self.vocab) abstract_withunks = data.show_abs_oovs(original_abstract, self.vocab, (batch.art_oovs[0] if FLAGS.pointer else None)) best_hyp = beam_search.run_beam_search(self.sess, self.model, self.vocab, batch) output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self.vocab, (batch.art_oovs[0] if FLAGS.pointer else None)) try: fst_stop_idx = decoded_words.index(data.DECODING_END) decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) if FLAGS.onetime: self._write_for_rouge(original_abstract, decoded_words, counter) counter += 1 else: print("") print('INFO: ARTICLE: {}'.format(article_withunks)) print('INFO: REFERENCE SUMMARY: {}'.format(abstract_withunks)) print('INFO: GENERATED SUMMARY: {}'.format(decoded_output)) print("") self._write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.pointers) t1 = time.time() if t1-t0 > SECS_UNTIL_NEW_CKPT: print('INFO: Decoding for {} seconds, loading new checkpoint'.format(t1-t0)) while True: try: ckpt_state = tf.train.get_checkpoint_state(train_dir) print('INFO: Loading checkpoint {}'.format(ckpt_state.model_checkpoint_path)) self.saver.restore(self.sess, ckpt_state.model_checkpoint_path) break except: print('ERROR: Failed to restore checkpoint: {}, sleep for {} secs'.format(train_dir, 10)) time.sleep(10) t0 = time.time()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = FLAGS.decode_after while True: tf.reset_default_graph() batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") tf.logging.info("Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis if FLAGS.ac_training: best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch, self._dqn, self._dqn_sess, self._dqn_graph) else: best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: self.write_for_rouge(original_abstract_sents, decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1-t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1-t0) _ = util.load_ckpt(self._saver, self._sess, FLAGS.decode_from) t0 = time.time()
def decode2(self, batcher): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 while True: batch = batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen return decoded_output
def decode_one_batch(self, batch, withRouge=True): original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings original_uuid = batch.uuids[0] # string article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: # write ref summary and decoded summary to file, to eval with pyrouge later # self.write_for_rouge(original_abstract_sents, decoded_words,self._counter) self.write_for_flink(original_uuid, original_article, decoded_words, original_abstract_sents) self._counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool
def run_beam_decoder(self,batch): original_article=batch.original_articles[0] original_abstract=batch.original_abstracts[0] article_withunks=data.show_art_oovs(original_article,self.vocab) abstract_withunks=data.show_abs_oovs(original_abstract,self.vocab,(batch.art_oovs[0] if self.hp.pointer_gen==True else None)) best_list=beam_search.run_beam_search(self.sess,self.model,self.vocab,batch) print(best_list.tokens) output_ids=[int(t) for t in best_list.tokens[1:]] decoded_words=data.outputids2words(output_ids,self.vocab,(batch.art_oovs[0] if self.hp.pointer_gen==True else None)) print(decoded_words) try: fst_stop_idx=decoded_words.index(data.STOP_DECODING) decoded_words=decoded_words[:fst_stop_idx] except ValueError: decoded_words=decoded_words decoded_output=' '.join(decoded_words) return article_withunks,abstract_withunks,decoded_output
def run_flip(model, batcher, vocab): """Repeatedly runs eval iterations, logging to screen and writing summaries. Saves the model with the best loss seen so far.""" model.build_graph() # build the graph saver = tf.train.Saver(max_to_keep=3) # we will keep 3 best checkpoints at a time sess = tf.Session(config=util.get_config()) while True: _ = util.load_ckpt(saver, sess) # load a new checkpoint batch = batcher.next_batch() # get the next batch # run eval on the batch t0=time.time() # results = model.run_eval_step(sess, batch) results = model.run_decode(sess, batch) t1=time.time() tf.logging.info('seconds for batch: %.2f', t1-t0) # output_ids = [int(t) for t in best_hyp.tokens[1:]] # decoded_words = data.outputids2words(output_ids, self._vocab, ( # batch.art_oovs[0] if FLAGS.pointer_gen else None)) stop_id = model._vocab.word2id('[STOP]') for b in range(len(batch.original_abstracts)): original_abstract = batch.original_abstracts[b] # string fst_stop_idx = np.where(batch.target_batch[b,:] == stop_id)[0] if len(fst_stop_idx): fst_stop_idx = fst_stop_idx[0] else: fst_stop_idx = len(batch.target_batch[b,:]) abstract_withunks = data.show_abs_oovs(original_abstract, model._vocab, None) # string tf.logging.info('REFERENCE SUMMARY: %s', abstract_withunks) output_ids = [int(x) for x in results['ids'][:, b, 0]] output_ids = output_ids[:fst_stop_idx] decoded_words = data.outputids2words(output_ids, model._vocab, None) decoded_output = ' '.join(decoded_words) tf.logging.info('GENERATED SUMMARY: %s', decoded_output) loss = results['loss'] tf.logging.info('loss %.2f max flip %d dec %d target %d', loss, max(output_ids), batch.dec_batch.max(), batch.target_batch.max())
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" # t0 = time.time() batch = self._batcher.next_batch() # 1 example repeated across batch original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string # input data article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string # tf.logging.info('ARTICLE: %s', article) # tf.logging.info('GENERATED SUMMARY: %s', decoded_output) sys.stdout.write(decoded_output)
def decode(self, batches): counter = 0 for batch in batches: if (counter < 10000): original_article = batch.original_articles[0] original_abstract = batch.original_abstracts[0] original_abstract_sents = batch.original_abstracts_sents[0] article_withunks = data.show_art_oovs(original_article, self._vocab) abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, batch.art_oovs[0]) best_hypothesis = beam_search.run_beam_search( self._session, self._model, self._vocab, batch, self._hps) output_ids = [int(t) for t in best_hypothesis.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, batch.art_oovs[0]) try: fst_stop_idx = decoded_words.index( data.STOP_DECODING ) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string self.write_for_rouge(original_abstract_sents, decoded_words, counter, original_article) counter += 1 else: break self.rouge_eval()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" if not FLAGS.generate: t0 = time.time() counter = 0 while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass." ) tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING ) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: # write ref summary and decoded summary to file, to eval with pyrouge later self.write_for_rouge(original_abstract_sents, decoded_words, counter) counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time() # when generate=True else: counter = 0 while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass." ) return original_article = batch.original_articles[0] # string # original_abstract = batch.original_abstracts[0] # string # original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string # abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING ) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string counter += 1 # log output to screen print( "---------------------------------------------------------------------------" ) tf.logging.info('ARTICLE: %s', article_withunks) tf.logging.info('GENERATED SUMMARY: %s', decoded_output) print( "---------------------------------------------------------------------------" ) # self.write_for_rouge(original_abstract_sents, decoded_words, counter) # Write to file decoded_sents = [] while len(decoded_words) > 0: try: fst_period_idx = decoded_words.index(".") except ValueError: # there is text remaining that doesn't end in "." fst_period_idx = len(decoded_words) sent = decoded_words[:fst_period_idx + 1] # sentence up to and including the period decoded_words = decoded_words[fst_period_idx + 1:] # everything else decoded_sents.append(' '.join(sent)) # pyrouge calls a perl script that puts the data into HTML files. # Therefore we need to make our output HTML safe. decoded_sents = [make_html_safe(w) for w in decoded_sents] # Write to file result_file = os.path.join(self._result_dir, "%06d_summary.txt" % counter) with open(result_file, "w") as f: for idx, sent in enumerate(decoded_sents): f.write(sent) if idx == len( decoded_sents) - 1 else f.write(sent + "\n")
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 if FLAGS.decode_bleu: ref_file = os.path.join(self._bleu_dec_dir, "reference.txt") decoded_file = os.path.join(self._bleu_dec_dir, "decoded.txt") if os.path.exists(decoded_file): tf.logging.info('正在删除 %s', decoded_file) os.remove(decoded_file) if os.path.exists(ref_file): tf.logging.info('正在删除 %s', ref_file) os.remove(ref_file) while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") if FLAGS.decode_rouge: tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) try: t0 = time.time() results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) t1 = time.time() tf.logging.info( 'calculate Rouge score cost %d seconds', t1 - t0) except Exception as e: tf.logging.error('计算ROUGE出错 %s', e) if FLAGS.decode_bleu: ref_file = os.path.join(self._bleu_dec_dir, "reference.txt") decoded_file = os.path.join(self._bleu_dec_dir, "decoded.txt") t0 = time.time() bleu, bleu1, bleu2, bleu3, bleu4 = calcu_bleu( decoded_file, ref_file) sys_bleu = sys_bleu_file(decoded_file, ref_file) sys_bleu_perl = sys_bleu_perl_file(decoded_file, ref_file) t1 = time.time() tf.logging.info(bcolors.HEADER + '-----------BLEU SCORE-----------' + bcolors.ENDC) tf.logging.info( bcolors.OKGREEN + '%f \t %f \t %f \t %f \t %f' + bcolors.ENDC, bleu, bleu1, bleu2, bleu3, bleu4) tf.logging.info( bcolors.OKGREEN + 'sys_bleu %f' + bcolors.ENDC, sys_bleu) tf.logging.info( bcolors.OKGREEN + 'sys_bleu_perl %s' + bcolors.ENDC, sys_bleu_perl) tf.logging.info(bcolors.HEADER + '-----------BLEU SCORE-----------' + bcolors.ENDC) tf.logging.info('calculate BLEU score cost %d seconds', t1 - t0) break original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ''.join(decoded_words) # single string if FLAGS.single_pass: print_results(article_withunks, abstract_withunks, decoded_output, counter) # log output to screen if FLAGS.decode_rouge: self.write_for_rouge( original_abstract_sents, decoded_words, counter ) # write ref summary and decoded summary to file, to eval with pyrouge later if FLAGS.decode_bleu: self.write_for_bleu(original_abstract_sents, decoded_words) counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 # while True: batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") tf.logging.info("Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string article_withunks = re.sub(" ","",article_withunks) # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words for w in range(0,len(decoded_words)): decoded_words[w]=str(decoded_words[w]) decoded_output = ' '.join(decoded_words) # single string decoded_output = re.sub("[UNK+(+)+\ ]","",decoded_output) decoded_output = re.sub("\.","",decoded_output) if "2002" not in decoded_output: print("?") decoded_output = re.sub("2","",decoded_output) # decoded_output = re.sub(" ","",decoded_output) decoded_output = re.sub("\[","",decoded_output) decoded_output = re.sub("\]","",decoded_output) # decoded_output = re.sub(" ","",decoded_output) decoded_output = re.sub("[养生+_社会_+社会频道+光明网+(+)+【+】+组图+万象+时尚]","",decoded_output) # decoded_output = re.sub("组图","",decoded_output) # decoded_output = re.sub("(","",decoded_output) # decoded_output = re.sub(")","",decoded_output) # decoded_output = re.sub("【","",decoded_output) # decoded_output = re.sub("】","",decoded_output) decoded_output = decoded_output.strip(decoded_output[-1]+decoded_output[-2]) decoded_words = decoded_output # print(type(decoded_output)) print(decoded_output) if FLAGS.single_pass: self.write_for_rouge(original_abstract_sents, decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1-t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1-t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self, output_dir=None): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 idx = 0 # used to store values during decoding in a list each outut_str = "" beam_search_str = "" metadata = [] # evaluate over a fixed number of test set while True: #idx <=100 : print("[%d]" % idx) batch = self._batcher.next_batch( ) # 1 example repeated across batch # if idx < 11000: # idx += 1 # continue if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get all the Hypothesis all_hyp = beam_search.run_beam_search( self._sess, self._model, self._vocab, batch, counter, self._lm_model, self._lm_word2idx, self._lm_idx2word ) #TODO changed the method signature just to look at the outputs of beam search if FLAGS.save_values: for h in all_hyp: output_ids = [int(t) for t in h.tokens[1:]] search_str = str( data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None))) beam_search_str += search_str beam_search_str += "\n" # Extract the get best Hypothesis best_hyp = all_hyp[0] # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) metadata.append(decoded_words) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string ########################### #print best hyp statistics hyp_stat = "" # log prob hyp_stat += "\navg log prob: %s.\n" % best_hyp.avg_log_prob # words overlap with article: this is buggy # tri, bi, uni = word_overlap.gram_search(ngrams(nltk.pos_tag(article_withunks.strip().split()), 3), ngrams(nltk.pos_tag(decoded_output.strip().split()), 3)) # hyp_stat += "trigram overlap: %s. bigram overlap: %s. unigram overlap: %s.\n"%(uni, bi, tri) print_statistics.get_overlap(article_withunks.strip(), decoded_output.strip(), match_count=self.overlap_dict) hyp_stat += "word overlap: " for key, value in self.overlap_dict.iteritems(): hyp_stat += "\n%d-gram avg overlap: %d" % (key, value / (counter + 1)) # num sentences and avg length self.total_nsentence += len(decoded_output.strip().split( ".")) #sentences are seperated by "." self.total_length += len(decoded_output.strip().split()) avg_nsentence, avg_length = self.total_nsentence / ( counter + 1), self.total_length / (counter + 1) hyp_stat += "\nnum sentences: %s. avg len: %s.\n" % (avg_nsentence, avg_length) # entropy?? if FLAGS.print_info: print(hyp_stat) ########################### # saves data into numpy files for analysis if FLAGS.save_values: save_decode_data.save_data_iteration(self._decode_dir, counter, best_hyp) if FLAGS.single_pass: #change to counter later self.write_for_rouge( original_abstract_sents, decoded_words, counter ) # write ref summary and decoded summary to file, to eval with pyrouge later # writing all the output combined to a file if FLAGS.print_info: output = '\nARTICLE: %s\n REFERENCE SUMMARY: %s\n' 'GENERATED SUMMARY: %s\n' % ( article_withunks, abstract_withunks, decoded_output) print(output) outut_str += output # Leena: modifying this to save more stuff self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, counter, best_hyp.log_prob, best_hyp.avg_log_prob, best_hyp.average_pgen) #change to counter later counter += 1 # this is how many examples we've decoded else: #Leena: I use the above condition so might have neglected making change to the below condition print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, counter) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time() idx += 1 #Leena: saving entire output and beam output as a string to write to a file if FLAGS.save_values: save_decode_data.save_data_once(self._decode_dir, FLAGS.result_path, outut_str, beam_search_str, metadata)
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) # results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) # rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings original_topic = batch.original_topics[0] article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis all_sencent = [] best_hyps = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) put_ids = [int(t) for t in best_hyps[0].tokens[1:]] standard_words = data.outputids2words( put_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Extract the output ids from the hypothesis and convert back to words score = [] for best_hyp in best_hyps: output_ids = [int(t) for t in best_hyp.tokens[1:]] tmp_decoded = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) all_sencent.extend(self.removes(tmp_decoded)) all_sencent.append('\n@next\n') score.append(self.get_score(original_topic, tmp_decoded)) all_sencent.extend(self.removes(standard_words)) all_sencent.append('\n@next\n') max_index = score.index(max(score)) put_ids = [int(t) for t in best_hyps[max_index].tokens[1:]] decoded_words = data.outputids2words( put_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) all_sencent.extend(self.removes(decoded_words)) decoded_words = all_sencent # Remove the [STOP] token from decoded_words, if necessary decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: self.write_for_rouge( original_abstract_sents, decoded_words, counter ) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.topic_attn_dists, best_hyp. p_gens) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) return original_reviews = batch.original_reviews[0] # list original_answer = batch.original_answers[0] # string original_answers_sents = batch.original_answers_sents[ 0] # list of strings original_question = batch.original_questions[0] y_target = batch.y_target_batch[0] review_withunks = [ data.show_art_oovs(original_review, self._vocab) for original_review in original_reviews ] # string question_withunks = data.show_art_oovs(original_question, self._vocab) # string answer_withunks = data.show_abs_oovs( original_answer, self._vocab, (batch.oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp, y_pred = beam_search.run_beam_search( self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: self.write_for_eval( original_answers_sents, decoded_words, original_question, y_target, y_pred, counter ) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(review_withunks, answer_withunks, decoded_output, original_question, y_target, y_pred) # log output to screen self.write_for_attnvis( question_withunks, review_withunks, decoded_words, best_hyp.r_attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self): """ Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals. """ counter = 0 scores = [] while True: batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir, ) tf.logging.info("Mean score: %s", sum(scores) / len(scores)) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, batch.art_oovs[0]) # string # Run beam search to get best Hypothesis t_beam = time.time() best_hyp, best_score = beam_search.run_beam_search( self._sess, self._model, self._vocab, batch, FLAGS.beam_size, FLAGS.max_dec_steps, FLAGS.min_dec_steps, FLAGS.trace_path ) scores.append(best_score) tf.logging.info("Time to decode one example: %f", time.time() - t_beam) tf.logging.info("Mean score: %s", sum(scores) / len(scores)) # Extract the output ids from the hypothesis and convert back to words decoded_words = best_hyp.token_strings[1:] # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: self.write_for_rouge(original_abstract, decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: # log output to screen print_results( article_withunks, abstract_withunks, decoded_output, best_hyp, [best_score] ) # write info to .json file for visualization tool self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, best_hyp.log_probs ) raw_input()
def decode(self): """ Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals """ t0 = time.time() start_time = t0 counter = 0 while True: # 1 example repeated across batch batch = self._batcher.next_batch() if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass, using %d seconds.", time.time() - start_time) tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) #todo: need to update for rouge # results_dict = rouge_eval(self._rouge_ref_dir, # self._rouge_dec_dir) # rouge_log(results_dict, self._decode_dir) return original_title = batch.original_titles[0] # string original_summarization = batch.original_summarizations[0] # string #original_abstract_sents = batch.original_abstracts_sents[ # 0] # list of strings title_withunks = data.show_art_oovs(original_title, self._vocab) abstract_withunks = data.show_abs_oovs( original_summarization, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: # index of the (first) [STOP] symbol fst_stop_idx = decoded_words.index(data.MARK_EOS) decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ''.join(decoded_words) # single string if FLAGS.single_pass: #todo: need to check # write ref summary and decoded summary to file, to eval with pyrouge later self.write_result(original_title, original_summarization, decoded_words, counter) # self.write_for_eval(original_summarization, output_ids, # counter) counter += 1 # this is how many examples we've decoded else: # log output to screen print_results(title_withunks, abstract_withunks, decoded_output) # write info to .json file for visualization tool self.write_for_attnvis(title_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists) # Check if SECS_UNTIL_NEW_CKPT has elapsed; # if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch #if counter >= 2: # batch = None if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") if FLAGS.eval_type == "cnn": tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) if FLAGS.eval_type == "medsumm": tf.logging.info( "Writing generated answer summaries to file...") with open(FLAGS.generated_data_file, "w", encoding="utf-8") as f: json.dump(self._generated_answers, f, indent=4) return question = batch.questions[0] original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: if FLAGS.eval_type == "medsumm": # Save generated data for answer summ evaluation: self.write_data_for_medsumm_eval( original_abstract_sents, decoded_words, question, counter ) # write ref summary and decoded summary to file, for later evaluation. tf.logging.info("saving summary %i", counter) if FLAGS.eval_type == "cnn": # write data for See's original evaluation done with pyrouge. self.write_for_rouge( original_abstract_sents, decoded_words, counter ) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 all_logits = [] all_one_hot_lables = [] while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) evaluation(all_logits, all_one_hot_lables, FLAGS.max_side_steps) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp, pic, sticker_logits = beam_search.run_beam_search( self._sess, self._model, self._vocab, batch) one_hot_targets = np.eye( FLAGS.max_side_steps)[batch.dec_pic_target] for i in range(FLAGS.batch_size): logits = sticker_logits[i].tolist() one_hot = one_hot_targets[i].tolist() for l, o in zip(logits, one_hot): all_logits.append(l) all_one_hot_lables.append(o) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: self.write_for_rouge( original_abstract_sents, decoded_words, counter, pic, batch.dec_pic_target[0] ) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode_iteratively(self, example_generator, total, names_to_types, ssi_list, hps): attn_vis_idx = 0 for example_idx, example in enumerate( tqdm(example_generator, total=total)): raw_article_sents, groundtruth_similar_source_indices_list, groundtruth_summary_text, corefs, groundtruth_article_lcs_paths_list = util.unpack_tf_example( example, names_to_types) article_sent_tokens = [ util.process_sent(sent) for sent in raw_article_sents ] groundtruth_summ_sents = [[ sent.strip() for sent in groundtruth_summary_text.strip().split('\n') ]] groundtruth_summ_sent_tokens = [ sent.split(' ') for sent in groundtruth_summ_sents[0] ] if ssi_list is None: # this is if we are doing the upper bound evaluation (ssi_list comes straight from the groundtruth) sys_ssi = groundtruth_similar_source_indices_list sys_alp_list = groundtruth_article_lcs_paths_list if FLAGS.singles_and_pairs == 'singles': sys_ssi = util.enforce_sentence_limit(sys_ssi, 1) sys_alp_list = util.enforce_sentence_limit(sys_alp_list, 1) elif FLAGS.singles_and_pairs == 'both': sys_ssi = util.enforce_sentence_limit(sys_ssi, 2) sys_alp_list = util.enforce_sentence_limit(sys_alp_list, 2) sys_ssi, sys_alp_list = util.replace_empty_ssis( sys_ssi, raw_article_sents, sys_alp_list=sys_alp_list) else: gt_ssi, sys_ssi, ext_len, sys_token_probs_list = ssi_list[ example_idx] sys_alp_list = ssi_functions.list_labels_from_probs( sys_token_probs_list, FLAGS.tag_threshold) if FLAGS.singles_and_pairs == 'singles': sys_ssi = util.enforce_sentence_limit(sys_ssi, 1) sys_alp_list = util.enforce_sentence_limit(sys_alp_list, 1) groundtruth_similar_source_indices_list = util.enforce_sentence_limit( groundtruth_similar_source_indices_list, 1) gt_ssi = util.enforce_sentence_limit(gt_ssi, 1) elif FLAGS.singles_and_pairs == 'both': sys_ssi = util.enforce_sentence_limit(sys_ssi, 2) sys_alp_list = util.enforce_sentence_limit(sys_alp_list, 2) groundtruth_similar_source_indices_list = util.enforce_sentence_limit( groundtruth_similar_source_indices_list, 2) gt_ssi = util.enforce_sentence_limit(gt_ssi, 2) # if gt_ssi != groundtruth_similar_source_indices_list: # raise Exception('Example %d has different groundtruth source indices: ' + str(groundtruth_similar_source_indices_list) + ' || ' + str(gt_ssi)) if FLAGS.dataset_name == 'xsum': sys_ssi = [sys_ssi[0]] final_decoded_words = [] final_decoded_outpus = '' best_hyps = [] highlight_html_total = '<u>System Summary</u><br><br>' for ssi_idx, ssi in enumerate(sys_ssi): # selected_article_lcs_paths = None selected_article_lcs_paths = sys_alp_list[ssi_idx] ssi, selected_article_lcs_paths = util.make_ssi_chronological( ssi, selected_article_lcs_paths) selected_article_lcs_paths = [selected_article_lcs_paths] selected_raw_article_sents = util.reorder( raw_article_sents, ssi) selected_article_text = ' '.join([ ' '.join(sent) for sent in util.reorder(article_sent_tokens, ssi) ]) selected_doc_indices_str = '0 ' * len( selected_article_text.split()) if FLAGS.upper_bound: selected_groundtruth_summ_sent = [[ groundtruth_summ_sents[0][ssi_idx] ]] else: selected_groundtruth_summ_sent = groundtruth_summ_sents batch = create_batch(selected_article_text, selected_groundtruth_summ_sent, selected_doc_indices_str, selected_raw_article_sents, selected_article_lcs_paths, FLAGS.batch_size, hps, self._vocab) original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # article_withunks = data.show_art_oovs(original_article, self._vocab) # string # abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string if FLAGS.first_intact and ssi_idx == 0: decoded_words = selected_article_text.strip().split() decoded_output = selected_article_text else: decoded_words, decoded_output, best_hyp = decode_example( self._sess, self._model, self._vocab, batch, example_idx, hps) best_hyps.append(best_hyp) final_decoded_words.extend(decoded_words) final_decoded_outpus += decoded_output if example_idx < 100 or (example_idx >= 2000 and example_idx < 2100): min_matched_tokens = 2 selected_article_sent_tokens = [ util.process_sent(sent) for sent in selected_raw_article_sents ] highlight_summary_sent_tokens = [decoded_words] highlight_ssi_list, lcs_paths_list, highlight_article_lcs_paths_list, highlight_smooth_article_lcs_paths_list = ssi_functions.get_simple_source_indices_list( highlight_summary_sent_tokens, selected_article_sent_tokens, None, 2, min_matched_tokens) highlighted_html = ssi_functions.html_highlight_sents_in_article( highlight_summary_sent_tokens, highlight_ssi_list, selected_article_sent_tokens, lcs_paths_list=lcs_paths_list, article_lcs_paths_list= highlight_smooth_article_lcs_paths_list) highlight_html_total += highlighted_html + '<br>' if FLAGS.attn_vis and example_idx < 200: self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, attn_vis_idx ) # write info to .json file for visualization tool attn_vis_idx += 1 if len(final_decoded_words) >= 100: break gt_ssi_list, gt_alp_list = util.replace_empty_ssis( groundtruth_similar_source_indices_list, raw_article_sents, sys_alp_list=groundtruth_article_lcs_paths_list) highlight_html_gt = '<u>Reference Summary</u><br><br>' for ssi_idx, ssi in enumerate(gt_ssi_list): selected_article_lcs_paths = gt_alp_list[ssi_idx] try: ssi, selected_article_lcs_paths = util.make_ssi_chronological( ssi, selected_article_lcs_paths) except: util.print_vars(ssi, example_idx, selected_article_lcs_paths) raise selected_raw_article_sents = util.reorder( raw_article_sents, ssi) if example_idx < 100 or (example_idx >= 2000 and example_idx < 2100): min_matched_tokens = 2 selected_article_sent_tokens = [ util.process_sent(sent) for sent in selected_raw_article_sents ] highlight_summary_sent_tokens = [ groundtruth_summ_sent_tokens[ssi_idx] ] highlight_ssi_list, lcs_paths_list, highlight_article_lcs_paths_list, highlight_smooth_article_lcs_paths_list = ssi_functions.get_simple_source_indices_list( highlight_summary_sent_tokens, selected_article_sent_tokens, None, 2, min_matched_tokens) highlighted_html = ssi_functions.html_highlight_sents_in_article( highlight_summary_sent_tokens, highlight_ssi_list, selected_article_sent_tokens, lcs_paths_list=lcs_paths_list, article_lcs_paths_list= highlight_smooth_article_lcs_paths_list) highlight_html_gt += highlighted_html + '<br>' if example_idx < 100 or (example_idx >= 2000 and example_idx < 2100): self.write_for_human(raw_article_sents, groundtruth_summ_sents, final_decoded_words, example_idx) highlight_html_total = ssi_functions.put_html_in_two_columns( highlight_html_total, highlight_html_gt) ssi_functions.write_highlighted_html(highlight_html_total, self._highlight_dir, example_idx) # if example_idx % 100 == 0: # attn_dir = os.path.join(self._decode_dir, 'attn_vis_data') # attn_selections.process_attn_selections(attn_dir, self._decode_dir, self._vocab) rouge_functions.write_for_rouge( groundtruth_summ_sents, None, example_idx, self._rouge_ref_dir, self._rouge_dec_dir, decoded_words=final_decoded_words, log=False ) # write ref summary and decoded summary to file, to eval with pyrouge later # if FLAGS.attn_vis: # self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, example_idx) # write info to .json file for visualization tool example_idx += 1 # this is how many examples we've decoded logging.info("Decoder has finished reading dataset for single_pass.") logging.info("Output has been saved in %s and %s.", self._rouge_ref_dir, self._rouge_dec_dir) if len(os.listdir(self._rouge_ref_dir)) != 0: if FLAGS.dataset_name == 'xsum': l_param = 100 else: l_param = 100 logging.info("Now starting ROUGE eval...") results_dict = rouge_functions.rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir, l_param=l_param) rouge_functions.rouge_log(results_dict, self._decode_dir)
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 f = os.path.join(FLAGS.log_root, "output.txt") # print("----------------"+f) outputfile = codecs.open(f, "w", "utf8") output_result = [] list_of_reference = [] while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode logging.info("eval_finished") outputfile.close() break print(self._batcher.c_index) original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis result, all_candidate = self.eval_one_batch_with_candidate( self._sess, self._model, self._vocab, batch) #result = self.eval_one_batch(self._sess, self._model, self._vocab, batch) for i, instance in enumerate(result): if i == len(batch.art_oovs): break if i >= batch.real_length: print("eval done with {} isntances".format( len(output_result))) break out_words = data.outputids2words(instance, self._model._vocab_out, batch.art_oovs[i]) if data.STOP_DECODING in out_words: out_words = out_words[:out_words.index(data.STOP_DECODING)] candidates_value = self.get_condidate_predicate( out_words, all_candidate[i], batch.art_oovs[i]) candidates_value = "_||_".join(candidates_value) output_now = " ".join(out_words) output_result.append(output_now) # refer = " ".join(refer) refer = batch.original_abstracts[i].strip() list_of_reference.append([refer]) outputfile.write(batch.original_articles[i] + '\t' + batch.original_abstracts[i] + '\t' + output_now + '\t' + candidates_value + '\n') bleu = matrix.bleu_score(list_of_reference, output_result) acc = matrix.compute_acc(list_of_reference, output_result) print("bleu : {} acc : {}".format(bleu, acc)) return
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 f = os.path.join(FLAGS.log_root, "output.txt") # print("----------------"+f) outputfile = codecs.open(f, "w", "utf8") while True: batch = self._batcher.next_single_decode_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") # tf.logging.info("Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) # results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) # rouge_log(results_dict, self._decode_dir) outputfile.close() return print(self._batcher.c_index) original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_k_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch, FLAGS.best_k_hyp) for best_hyp in best_k_hyp: # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string #if FLAGS.single_pass: if True: # print(original_article+"\t"+decoded_output + "\n") outputfile.write(original_article + "\t" + original_abstract + "\t" + decoded_output + "\t" + str( best_hyp.avg_log_prob) + "\t" + str(best_hyp.len) + "\n") # self.write_for_rouge(original_abstract_sents, decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded outputfile.flush() else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" # Return original articles and their summaries if in API mode: if FLAGS.api_mode: articles = [] summaries = [] summaries_tokens = [] t0 = time.time() counter = 0 while True: batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") tf.logging.info("Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) if not FLAGS.api_mode: results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return else: assert FLAGS.pickle_file, "Pickle path not specified" decoder_final_output = {'articles':articles, 'summaries':summaries, 'summaries_tokens': summaries_tokens} pickle.dump(decoder_final_output, open(FLAGS.pickle_file, "wb")) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.api_mode: # TODO add status printer: articles.append(original_article) summaries.append(decoded_output) summaries_tokens.append(decoded_words) counter += 1 if not counter % 25: print(f"{counter} articles summarized") elif FLAGS.single_pass: self.write_for_rouge(original_abstract_sents, decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1-t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1-t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 total = len(glob.glob(self._batcher._data_path)) * 1000 pbar = tqdm(total=total) while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" logging.info( "Decoder has finished reading dataset for single_pass.") logging.info("Output has been saved in %s and %s.", self._rouge_ref_dir, self._rouge_dec_dir) if len(os.listdir(self._rouge_ref_dir)) != 0: logging.info("Now starting ROUGE eval...") results_dict = rouge_functions.rouge_eval( self._rouge_ref_dir, self._rouge_dec_dir) rouge_functions.rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string all_original_abstract_sents = batch.all_original_abstracts_sents[0] raw_article_sents = batch.raw_article_sents[0] article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string decoded_words, decoded_output, best_hyp = decode_example( self._sess, self._model, self._vocab, batch, counter, self._batcher._hps) if FLAGS.single_pass: if counter < 1000: self.write_for_human(raw_article_sents, all_original_abstract_sents, decoded_words, counter) rouge_functions.write_for_rouge( all_original_abstract_sents, None, counter, self._rouge_ref_dir, self._rouge_dec_dir, decoded_words=decoded_words ) # write ref summary and decoded summary to file, to eval with pyrouge later if FLAGS.attn_vis: self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, counter ) # write info to .json file for visualization tool counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, counter) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time() pbar.update(1) pbar.close()
def run_decode(model, batcher, vocab): print "build graph..." model.build_graph() saver = tf.train.Saver(max_to_keep=3) sess = tf.Session(config=util.get_config()) saver = tf.train.Saver() ckpt_path = util.load_ckpt(saver, sess) if FLAGS.single_pass: ckpt_name = "ckpt-" + ckpt_path.split('-')[-1] dirname = "decode_maxenc_%ibeam_%imindec_%imaxdec_%i" % (FLAGS.max_enc_steps, FLAGS.beam_size, FLAGS.min_dec_steps, FLAGS.max_dec_steps) decode_dir = os.path.join(FLAGS.log_root, dirname + ckpt_name) if os.path.exists(decode_dir): raise Exception('single_pass decode directory %s should not exist', decode_dir) else: decode_dir = os.path.join(FLAGS.log_root, 'decode') if not os.path.exists(decode_dir): os.mkdir(decode_dir) if FLAGS.single_pass: rouge_ref_dir = os.path.join(decode_dir, "reference") if not os.path.exists(rouge_ref_dir): os.mkdir(rouge_ref_dir) rouge_dec_dir = os.path.join(decode_dir, "decoded") if not os.path.exists(rouge_dec_dir): os.mkdir(rouge_dec_dir) counter = 0 t0 = time.time() while True: batch = batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" print("Decoder has finished reading dataset for single_pass.") print("Output has been saved in %s and %s. Now starting ROUGE eval...", rouge_ref_dir, rouge_dec_dir) results_dict = rouge_eval(rouge_ref_dir, rouge_dec_dir) rouge_log(results_dict, decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, vocab, None) # string # Run beam search to get best Hypothesis output = model.run_beam_decode_step(sess, batch, vocab) output_ids = [int(t) for t in output] decoded_words = data.outputids2words(output_ids, vocab, None) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: write_for_rouge(original_abstract_sents, decoded_words, counter, rouge_ref_dir, rouge_dec_dir) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(saver, sess) t0 = time.time()