def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 while True: batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") tf.logging.info("Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[0] # list of strings original_category = batch.categoryWords[0] original_tags = batch.tagWords[0] article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string category_withunks = data.show_art_oovs(original_category, self._vocab) # strin # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: # write ref summary and decoded summary to file, to eval with pyrouge later self.write_for_rouge(original_article, original_abstract_sents, decoded_words, original_tags,original_category, counter) counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output, original_category) # log output to screen self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def process_one_article(self, original_article_sents, original_abstract_sents, \ original_selected_ids, output_ids, oovs, \ attn_dists, p_gens, log_probs, counter): # Remove the [STOP] token from decoded_words, if necessary decoded_words = data.outputids2words(output_ids, self._vocab, oovs) try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string decoded_sents = data.words2sents(decoded_words) if FLAGS.single_pass: verbose = False if FLAGS.mode == 'eval' else True self.write_for_rouge(original_abstract_sents, decoded_sents, counter, verbose) # write ref summary and decoded summary to file, to eval with pyrouge later if FLAGS.decode_method == 'beam' and FLAGS.save_vis: original_article = ' '.join(original_article_sents) original_abstract = ' '.join(original_abstract_sents) article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, oovs) self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, \ attn_dists, p_gens, log_probs, counter, verbose) if FLAGS.save_pkl: self.save_result(original_article_sents, original_abstract_sents, \ original_selected_ids, decoded_sents, counter, verbose)
def decodeOneSample(self, batches): batch = batches[0] original_article = batch.original_articles[0] original_abstract = batch.original_abstracts[0] original_abstract_sents = batch.original_abstracts_sents[0] article_withunks = data.show_art_oovs(original_article, self._vocab) abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, batch.art_oovs[0]) best_hypothesis = beam_search.run_beam_search(self._session, self._model, self._vocab, batch, self._hps) output_ids = [int(t) for t in best_hypothesis.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, batch.art_oovs[0]) try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string self.write_for_rouge(original_abstract_sents, decoded_words, 0, original_article) self.rouge_eval() print_results(article_withunks, abstract_withunks, decoded_output) self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hypothesis.attn_dists, best_hypothesis.p_gens)
def decode(self): decode_relations = [] original_relations = [] while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode # todo after finish final batch, compute precision, recall and f1_score p, r, f1 = calculate_measure_cmp(decode_relations, original_relations) tf.logging.info("p: %.4f, r: %.4f, f1: %.4f", p, r, f1) assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words_webnlg( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string decode_relations.append(decoded_output) original_relations.append(original_abstract) print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool
def decode(self, ckpt_file=None): FLAGS = self._FLAGS # load latest checkpoint misc_utils.load_ckpt(self._saver, self._sess, self._ckpt_dir, ckpt_file) counter = 0 f = open(self._decode_dir, "w") while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode tf.logging.info( "Decoder has finished reading dataset for single_pass.") break original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch, FLAGS) # Extract the output ids from the hypothesis and convert back to # words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: # index of the (first) [STOP] symbol fst_stop_idx = decoded_words.index(data.STOP_DECODING) decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words # write ref summary and decoded summary to file, to eval with # pyrouge later # self.write_for_rouge(original_abstract_sents, decoded_words, counter) processed = self.depreciated_processing(decoded_words) f.write(processed + "\n") counter += 1 if counter % 100 == 0: print("%d sentences decoded" % counter) f.close()
def pair_wise_decode(self): f = os.path.join(FLAGS.data_path, "output.txt") outputfile = codecs.open(f, "w", "utf8") output_result = [] list_of_reference = [] while True: batch = self._batcher.next_pairwised_decode_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode logging.info("eval_finished") outputfile.close() break print(self._batcher.c_index) original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis result = self.eval_one_batch(self._sess, self._model, self._vocab, batch) # result = self.eval_one_batch(self._sess, self._model, self._vocab, batch) for i, instance in enumerate(result): if i == len(batch.art_oovs): break if i >= batch.real_length: print("eval done with {} isntances".format( len(output_result))) break out_words = data.outputids2words(instance, self._model._vocab_out, batch.art_oovs[i]) if data.STOP_DECODING in out_words: out_words = out_words[:out_words.index(data.STOP_DECODING)] output_now = " ".join(out_words) output_result.append(output_now) # refer = " ".join(refer) refer = batch.original_abstracts[i].strip() list_of_reference.append([refer]) outputfile.write(batch.original_articles[i] + '\t' + output_now + '\n') bleu = matrix.bleu_score(list_of_reference, output_result) acc = matrix.compute_acc(list_of_reference, output_result) print("bleu : {} acc : {}".format(bleu, acc)) return
def _decode(self): """ """ t0 = time.time() counter = 0 while True: batch = self.batcher._next_batch() if batch is None: assert FLAGS.onetime, "Dataset exhausted, but we are not in onetime mode" print('INFO: Decoder has finished reading dataset for onetime.') print('INFO: Output has been saved in {} and {}, start ROUGE eval...'.format(self.rouge_ref_dir, self.rouge_dec_dir)) results_dict = rouge_eval(self.rouge_ref_dir, self.rouge_dec_dir) rouge_log(results_dict, self.decode_dir) return original_article = batch.original_articles[0] original_abstract = batch.original_abstracts[0] article_withunks = data.show_art_oovs(original_article, self.vocab) abstract_withunks = data.show_abs_oovs(original_abstract, self.vocab, (batch.art_oovs[0] if FLAGS.pointer else None)) best_hyp = beam_search.run_beam_search(self.sess, self.model, self.vocab, batch) output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self.vocab, (batch.art_oovs[0] if FLAGS.pointer else None)) try: fst_stop_idx = decoded_words.index(data.DECODING_END) decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) if FLAGS.onetime: self._write_for_rouge(original_abstract, decoded_words, counter) counter += 1 else: print("") print('INFO: ARTICLE: {}'.format(article_withunks)) print('INFO: REFERENCE SUMMARY: {}'.format(abstract_withunks)) print('INFO: GENERATED SUMMARY: {}'.format(decoded_output)) print("") self._write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.pointers) t1 = time.time() if t1-t0 > SECS_UNTIL_NEW_CKPT: print('INFO: Decoding for {} seconds, loading new checkpoint'.format(t1-t0)) while True: try: ckpt_state = tf.train.get_checkpoint_state(train_dir) print('INFO: Loading checkpoint {}'.format(ckpt_state.model_checkpoint_path)) self.saver.restore(self.sess, ckpt_state.model_checkpoint_path) break except: print('ERROR: Failed to restore checkpoint: {}, sleep for {} secs'.format(train_dir, 10)) time.sleep(10) t0 = time.time()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 out_num=0 summaries=[] while True: batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert self.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") ##I commented those lines # tf.logging.info("Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) # results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) # rouge_log(results_dict, self._decode_dir) print out_num return summaries original_article = batch.original_articles[0] # string # I commented those lines # original_abstract = batch.original_abstracts[0] # string # original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string # I commented this line # abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch,self.beam_size,self.max_dec_steps,self.min_dec_steps) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if self.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if self.single_pass: summaries.append(decoded_output) # open('s'+str(out_num)+'.txt','w').write(decoded_output) # with open('output'+str(out_num)+'.txt','w') as output: # output.write(original_article+'\n*******************************************\n\n'+decoded_output) out_num+=1 print out_num #this line is commented by me # self.write_for_rouge(original_abstract_sents, decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = FLAGS.decode_after while True: tf.reset_default_graph() batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") tf.logging.info("Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis if FLAGS.ac_training: best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch, self._dqn, self._dqn_sess, self._dqn_graph) else: best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: self.write_for_rouge(original_abstract_sents, decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1-t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1-t0) _ = util.load_ckpt(self._saver, self._sess, FLAGS.decode_from) t0 = time.time()
def decode2(self, batcher): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 while True: batch = batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen return decoded_output
def decode_one_batch(self, batch, withRouge=True): original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings original_uuid = batch.uuids[0] # string article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: # write ref summary and decoded summary to file, to eval with pyrouge later # self.write_for_rouge(original_abstract_sents, decoded_words,self._counter) self.write_for_flink(original_uuid, original_article, decoded_words, original_abstract_sents) self._counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 while True: batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode tf.logging.info("Decoder has finished reading dataset for single_pass.") return original_article = batch.original_articles[0] # string article_withunks = data.show_art_oovs(original_article, self._vocab) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: self.write_for_rouge(decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, decoded_output) # log output to screen # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1-t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1-t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def run_beam_decoder(self,batch): original_article=batch.original_articles[0] original_abstract=batch.original_abstracts[0] article_withunks=data.show_art_oovs(original_article,self.vocab) abstract_withunks=data.show_abs_oovs(original_abstract,self.vocab,(batch.art_oovs[0] if self.hp.pointer_gen==True else None)) best_list=beam_search.run_beam_search(self.sess,self.model,self.vocab,batch) print(best_list.tokens) output_ids=[int(t) for t in best_list.tokens[1:]] decoded_words=data.outputids2words(output_ids,self.vocab,(batch.art_oovs[0] if self.hp.pointer_gen==True else None)) print(decoded_words) try: fst_stop_idx=decoded_words.index(data.STOP_DECODING) decoded_words=decoded_words[:fst_stop_idx] except ValueError: decoded_words=decoded_words decoded_output=' '.join(decoded_words) return article_withunks,abstract_withunks,decoded_output
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" # t0 = time.time() batch = self._batcher.next_batch() # 1 example repeated across batch original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string # input data article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string # tf.logging.info('ARTICLE: %s', article) # tf.logging.info('GENERATED SUMMARY: %s', decoded_output) sys.stdout.write(decoded_output)
def decode(self, batches): counter = 0 for batch in batches: if (counter < 10000): original_article = batch.original_articles[0] original_abstract = batch.original_abstracts[0] original_abstract_sents = batch.original_abstracts_sents[0] article_withunks = data.show_art_oovs(original_article, self._vocab) abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, batch.art_oovs[0]) best_hypothesis = beam_search.run_beam_search( self._session, self._model, self._vocab, batch, self._hps) output_ids = [int(t) for t in best_hypothesis.tokens[1:]] decoded_words = data.outputids2words(output_ids, self._vocab, batch.art_oovs[0]) try: fst_stop_idx = decoded_words.index( data.STOP_DECODING ) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string self.write_for_rouge(original_abstract_sents, decoded_words, counter, original_article) counter += 1 else: break self.rouge_eval()
def decode(self): """ Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals. """ counter = 0 scores = [] while True: batch = self._batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info("Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir, ) tf.logging.info("Mean score: %s", sum(scores) / len(scores)) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, batch.art_oovs[0]) # string # Run beam search to get best Hypothesis t_beam = time.time() best_hyp, best_score = beam_search.run_beam_search( self._sess, self._model, self._vocab, batch, FLAGS.beam_size, FLAGS.max_dec_steps, FLAGS.min_dec_steps, FLAGS.trace_path ) scores.append(best_score) tf.logging.info("Time to decode one example: %f", time.time() - t_beam) tf.logging.info("Mean score: %s", sum(scores) / len(scores)) # Extract the output ids from the hypothesis and convert back to words decoded_words = best_hyp.token_strings[1:] # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: self.write_for_rouge(original_abstract, decoded_words, counter) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: # log output to screen print_results( article_withunks, abstract_withunks, decoded_output, best_hyp, [best_score] ) # write info to .json file for visualization tool self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, best_hyp.log_probs ) raw_input()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" if not FLAGS.generate: t0 = time.time() counter = 0 while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass." ) tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING ) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: # write ref summary and decoded summary to file, to eval with pyrouge later self.write_for_rouge(original_abstract_sents, decoded_words, counter) counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time() # when generate=True else: counter = 0 while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass." ) return original_article = batch.original_articles[0] # string # original_abstract = batch.original_abstracts[0] # string # original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string # abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING ) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string counter += 1 # log output to screen print( "---------------------------------------------------------------------------" ) tf.logging.info('ARTICLE: %s', article_withunks) tf.logging.info('GENERATED SUMMARY: %s', decoded_output) print( "---------------------------------------------------------------------------" ) # self.write_for_rouge(original_abstract_sents, decoded_words, counter) # Write to file decoded_sents = [] while len(decoded_words) > 0: try: fst_period_idx = decoded_words.index(".") except ValueError: # there is text remaining that doesn't end in "." fst_period_idx = len(decoded_words) sent = decoded_words[:fst_period_idx + 1] # sentence up to and including the period decoded_words = decoded_words[fst_period_idx + 1:] # everything else decoded_sents.append(' '.join(sent)) # pyrouge calls a perl script that puts the data into HTML files. # Therefore we need to make our output HTML safe. decoded_sents = [make_html_safe(w) for w in decoded_sents] # Write to file result_file = os.path.join(self._result_dir, "%06d_summary.txt" % counter) with open(result_file, "w") as f: for idx, sent in enumerate(decoded_sents): f.write(sent) if idx == len( decoded_sents) - 1 else f.write(sent + "\n")
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 if FLAGS.decode_bleu: ref_file = os.path.join(self._bleu_dec_dir, "reference.txt") decoded_file = os.path.join(self._bleu_dec_dir, "decoded.txt") if os.path.exists(decoded_file): tf.logging.info('正在删除 %s', decoded_file) os.remove(decoded_file) if os.path.exists(ref_file): tf.logging.info('正在删除 %s', ref_file) os.remove(ref_file) while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") if FLAGS.decode_rouge: tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) try: t0 = time.time() results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) t1 = time.time() tf.logging.info( 'calculate Rouge score cost %d seconds', t1 - t0) except Exception as e: tf.logging.error('计算ROUGE出错 %s', e) if FLAGS.decode_bleu: ref_file = os.path.join(self._bleu_dec_dir, "reference.txt") decoded_file = os.path.join(self._bleu_dec_dir, "decoded.txt") t0 = time.time() bleu, bleu1, bleu2, bleu3, bleu4 = calcu_bleu( decoded_file, ref_file) sys_bleu = sys_bleu_file(decoded_file, ref_file) sys_bleu_perl = sys_bleu_perl_file(decoded_file, ref_file) t1 = time.time() tf.logging.info(bcolors.HEADER + '-----------BLEU SCORE-----------' + bcolors.ENDC) tf.logging.info( bcolors.OKGREEN + '%f \t %f \t %f \t %f \t %f' + bcolors.ENDC, bleu, bleu1, bleu2, bleu3, bleu4) tf.logging.info( bcolors.OKGREEN + 'sys_bleu %f' + bcolors.ENDC, sys_bleu) tf.logging.info( bcolors.OKGREEN + 'sys_bleu_perl %s' + bcolors.ENDC, sys_bleu_perl) tf.logging.info(bcolors.HEADER + '-----------BLEU SCORE-----------' + bcolors.ENDC) tf.logging.info('calculate BLEU score cost %d seconds', t1 - t0) break original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis best_hyp = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ''.join(decoded_words) # single string if FLAGS.single_pass: print_results(article_withunks, abstract_withunks, decoded_output, counter) # log output to screen if FLAGS.decode_rouge: self.write_for_rouge( original_abstract_sents, decoded_words, counter ) # write ref summary and decoded summary to file, to eval with pyrouge later if FLAGS.decode_bleu: self.write_for_bleu(original_abstract_sents, decoded_words) counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens ) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self, output_dir=None): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 idx = 0 # used to store values during decoding in a list each outut_str = "" beam_search_str = "" metadata = [] # evaluate over a fixed number of test set while True: #idx <=100 : print("[%d]" % idx) batch = self._batcher.next_batch( ) # 1 example repeated across batch # if idx < 11000: # idx += 1 # continue if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get all the Hypothesis all_hyp = beam_search.run_beam_search( self._sess, self._model, self._vocab, batch, counter, self._lm_model, self._lm_word2idx, self._lm_idx2word ) #TODO changed the method signature just to look at the outputs of beam search if FLAGS.save_values: for h in all_hyp: output_ids = [int(t) for t in h.tokens[1:]] search_str = str( data.outputids2words(output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None))) beam_search_str += search_str beam_search_str += "\n" # Extract the get best Hypothesis best_hyp = all_hyp[0] # Extract the output ids from the hypothesis and convert back to words output_ids = [int(t) for t in best_hyp.tokens[1:]] decoded_words = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) metadata.append(decoded_words) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index( data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string ########################### #print best hyp statistics hyp_stat = "" # log prob hyp_stat += "\navg log prob: %s.\n" % best_hyp.avg_log_prob # words overlap with article: this is buggy # tri, bi, uni = word_overlap.gram_search(ngrams(nltk.pos_tag(article_withunks.strip().split()), 3), ngrams(nltk.pos_tag(decoded_output.strip().split()), 3)) # hyp_stat += "trigram overlap: %s. bigram overlap: %s. unigram overlap: %s.\n"%(uni, bi, tri) print_statistics.get_overlap(article_withunks.strip(), decoded_output.strip(), match_count=self.overlap_dict) hyp_stat += "word overlap: " for key, value in self.overlap_dict.iteritems(): hyp_stat += "\n%d-gram avg overlap: %d" % (key, value / (counter + 1)) # num sentences and avg length self.total_nsentence += len(decoded_output.strip().split( ".")) #sentences are seperated by "." self.total_length += len(decoded_output.strip().split()) avg_nsentence, avg_length = self.total_nsentence / ( counter + 1), self.total_length / (counter + 1) hyp_stat += "\nnum sentences: %s. avg len: %s.\n" % (avg_nsentence, avg_length) # entropy?? if FLAGS.print_info: print(hyp_stat) ########################### # saves data into numpy files for analysis if FLAGS.save_values: save_decode_data.save_data_iteration(self._decode_dir, counter, best_hyp) if FLAGS.single_pass: #change to counter later self.write_for_rouge( original_abstract_sents, decoded_words, counter ) # write ref summary and decoded summary to file, to eval with pyrouge later # writing all the output combined to a file if FLAGS.print_info: output = '\nARTICLE: %s\n REFERENCE SUMMARY: %s\n' 'GENERATED SUMMARY: %s\n' % ( article_withunks, abstract_withunks, decoded_output) print(output) outut_str += output # Leena: modifying this to save more stuff self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, counter, best_hyp.log_prob, best_hyp.avg_log_prob, best_hyp.average_pgen) #change to counter later counter += 1 # this is how many examples we've decoded else: #Leena: I use the above condition so might have neglected making change to the below condition print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, counter) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time() idx += 1 #Leena: saving entire output and beam output as a string to write to a file if FLAGS.save_values: save_decode_data.save_data_once(self._decode_dir, FLAGS.result_path, outut_str, beam_search_str, metadata)
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" tf.logging.info( "Decoder has finished reading dataset for single_pass.") tf.logging.info( "Output has been saved in %s and %s. Now starting ROUGE eval...", self._rouge_ref_dir, self._rouge_dec_dir) # results_dict = rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir) # rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings original_topic = batch.original_topics[0] article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis all_sencent = [] best_hyps = beam_search.run_beam_search(self._sess, self._model, self._vocab, batch) put_ids = [int(t) for t in best_hyps[0].tokens[1:]] standard_words = data.outputids2words( put_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # Extract the output ids from the hypothesis and convert back to words score = [] for best_hyp in best_hyps: output_ids = [int(t) for t in best_hyp.tokens[1:]] tmp_decoded = data.outputids2words( output_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) all_sencent.extend(self.removes(tmp_decoded)) all_sencent.append('\n@next\n') score.append(self.get_score(original_topic, tmp_decoded)) all_sencent.extend(self.removes(standard_words)) all_sencent.append('\n@next\n') max_index = score.index(max(score)) put_ids = [int(t) for t in best_hyps[max_index].tokens[1:]] decoded_words = data.outputids2words( put_ids, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) all_sencent.extend(self.removes(decoded_words)) decoded_words = all_sencent # Remove the [STOP] token from decoded_words, if necessary decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: self.write_for_rouge( original_abstract_sents, decoded_words, counter ) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.topic_attn_dists, best_hyp. p_gens) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time()
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 total = len(glob.glob(self._batcher._data_path)) * 1000 pbar = tqdm(total=total) while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" logging.info( "Decoder has finished reading dataset for single_pass.") logging.info("Output has been saved in %s and %s.", self._rouge_ref_dir, self._rouge_dec_dir) if len(os.listdir(self._rouge_ref_dir)) != 0: logging.info("Now starting ROUGE eval...") results_dict = rouge_functions.rouge_eval( self._rouge_ref_dir, self._rouge_dec_dir) rouge_functions.rouge_log(results_dict, self._decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string all_original_abstract_sents = batch.all_original_abstracts_sents[0] raw_article_sents = batch.raw_article_sents[0] article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string decoded_words, decoded_output, best_hyp = decode_example( self._sess, self._model, self._vocab, batch, counter, self._batcher._hps) if FLAGS.single_pass: if counter < 1000: self.write_for_human(raw_article_sents, all_original_abstract_sents, decoded_words, counter) rouge_functions.write_for_rouge( all_original_abstract_sents, None, counter, self._rouge_ref_dir, self._rouge_dec_dir, decoded_words=decoded_words ) # write ref summary and decoded summary to file, to eval with pyrouge later if FLAGS.attn_vis: self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, counter ) # write info to .json file for visualization tool counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, counter) # write info to .json file for visualization tool # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: logging.info( 'We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(self._saver, self._sess) t0 = time.time() pbar.update(1) pbar.close()
def run_decode(model, batcher, vocab): print "build graph..." model.build_graph() saver = tf.train.Saver(max_to_keep=3) sess = tf.Session(config=util.get_config()) saver = tf.train.Saver() ckpt_path = util.load_ckpt(saver, sess) if FLAGS.single_pass: ckpt_name = "ckpt-" + ckpt_path.split('-')[-1] dirname = "decode_maxenc_%ibeam_%imindec_%imaxdec_%i" % (FLAGS.max_enc_steps, FLAGS.beam_size, FLAGS.min_dec_steps, FLAGS.max_dec_steps) decode_dir = os.path.join(FLAGS.log_root, dirname + ckpt_name) if os.path.exists(decode_dir): raise Exception('single_pass decode directory %s should not exist', decode_dir) else: decode_dir = os.path.join(FLAGS.log_root, 'decode') if not os.path.exists(decode_dir): os.mkdir(decode_dir) if FLAGS.single_pass: rouge_ref_dir = os.path.join(decode_dir, "reference") if not os.path.exists(rouge_ref_dir): os.mkdir(rouge_ref_dir) rouge_dec_dir = os.path.join(decode_dir, "decoded") if not os.path.exists(rouge_dec_dir): os.mkdir(rouge_dec_dir) counter = 0 t0 = time.time() while True: batch = batcher.next_batch() # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode assert FLAGS.single_pass, "Dataset exhausted, but we are not in single_pass mode" print("Decoder has finished reading dataset for single_pass.") print("Output has been saved in %s and %s. Now starting ROUGE eval...", rouge_ref_dir, rouge_dec_dir) results_dict = rouge_eval(rouge_ref_dir, rouge_dec_dir) rouge_log(results_dict, decode_dir) return original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[0] # list of strings article_withunks = data.show_art_oovs(original_article, vocab) # string abstract_withunks = data.show_abs_oovs(original_abstract, vocab, None) # string # Run beam search to get best Hypothesis output = model.run_beam_decode_step(sess, batch, vocab) output_ids = [int(t) for t in output] decoded_words = data.outputids2words(output_ids, vocab, None) # Remove the [STOP] token from decoded_words, if necessary try: fst_stop_idx = decoded_words.index(data.STOP_DECODING) # index of the (first) [STOP] symbol decoded_words = decoded_words[:fst_stop_idx] except ValueError: decoded_words = decoded_words decoded_output = ' '.join(decoded_words) # single string if FLAGS.single_pass: write_for_rouge(original_abstract_sents, decoded_words, counter, rouge_ref_dir, rouge_dec_dir) # write ref summary and decoded summary to file, to eval with pyrouge later counter += 1 # this is how many examples we've decoded else: print_results(article_withunks, abstract_withunks, decoded_output) # log output to screen # Check if SECS_UNTIL_NEW_CKPT has elapsed; if so return so we can load a new checkpoint t1 = time.time() if t1 - t0 > SECS_UNTIL_NEW_CKPT: tf.logging.info('We\'ve been decoding with same checkpoint for %i seconds. Time to load new checkpoint', t1 - t0) _ = util.load_ckpt(saver, sess) t0 = time.time()
def decode_iteratively(self, example_generator, total, names_to_types, ssi_list, hps): attn_vis_idx = 0 for example_idx, example in enumerate( tqdm(example_generator, total=total)): raw_article_sents, groundtruth_similar_source_indices_list, groundtruth_summary_text, corefs, groundtruth_article_lcs_paths_list = util.unpack_tf_example( example, names_to_types) article_sent_tokens = [ util.process_sent(sent) for sent in raw_article_sents ] groundtruth_summ_sents = [[ sent.strip() for sent in groundtruth_summary_text.strip().split('\n') ]] groundtruth_summ_sent_tokens = [ sent.split(' ') for sent in groundtruth_summ_sents[0] ] if ssi_list is None: # this is if we are doing the upper bound evaluation (ssi_list comes straight from the groundtruth) sys_ssi = groundtruth_similar_source_indices_list sys_alp_list = groundtruth_article_lcs_paths_list if FLAGS.singles_and_pairs == 'singles': sys_ssi = util.enforce_sentence_limit(sys_ssi, 1) sys_alp_list = util.enforce_sentence_limit(sys_alp_list, 1) elif FLAGS.singles_and_pairs == 'both': sys_ssi = util.enforce_sentence_limit(sys_ssi, 2) sys_alp_list = util.enforce_sentence_limit(sys_alp_list, 2) sys_ssi, sys_alp_list = util.replace_empty_ssis( sys_ssi, raw_article_sents, sys_alp_list=sys_alp_list) else: gt_ssi, sys_ssi, ext_len, sys_token_probs_list = ssi_list[ example_idx] sys_alp_list = ssi_functions.list_labels_from_probs( sys_token_probs_list, FLAGS.tag_threshold) if FLAGS.singles_and_pairs == 'singles': sys_ssi = util.enforce_sentence_limit(sys_ssi, 1) sys_alp_list = util.enforce_sentence_limit(sys_alp_list, 1) groundtruth_similar_source_indices_list = util.enforce_sentence_limit( groundtruth_similar_source_indices_list, 1) gt_ssi = util.enforce_sentence_limit(gt_ssi, 1) elif FLAGS.singles_and_pairs == 'both': sys_ssi = util.enforce_sentence_limit(sys_ssi, 2) sys_alp_list = util.enforce_sentence_limit(sys_alp_list, 2) groundtruth_similar_source_indices_list = util.enforce_sentence_limit( groundtruth_similar_source_indices_list, 2) gt_ssi = util.enforce_sentence_limit(gt_ssi, 2) # if gt_ssi != groundtruth_similar_source_indices_list: # raise Exception('Example %d has different groundtruth source indices: ' + str(groundtruth_similar_source_indices_list) + ' || ' + str(gt_ssi)) if FLAGS.dataset_name == 'xsum': sys_ssi = [sys_ssi[0]] final_decoded_words = [] final_decoded_outpus = '' best_hyps = [] highlight_html_total = '<u>System Summary</u><br><br>' for ssi_idx, ssi in enumerate(sys_ssi): # selected_article_lcs_paths = None selected_article_lcs_paths = sys_alp_list[ssi_idx] ssi, selected_article_lcs_paths = util.make_ssi_chronological( ssi, selected_article_lcs_paths) selected_article_lcs_paths = [selected_article_lcs_paths] selected_raw_article_sents = util.reorder( raw_article_sents, ssi) selected_article_text = ' '.join([ ' '.join(sent) for sent in util.reorder(article_sent_tokens, ssi) ]) selected_doc_indices_str = '0 ' * len( selected_article_text.split()) if FLAGS.upper_bound: selected_groundtruth_summ_sent = [[ groundtruth_summ_sents[0][ssi_idx] ]] else: selected_groundtruth_summ_sent = groundtruth_summ_sents batch = create_batch(selected_article_text, selected_groundtruth_summ_sent, selected_doc_indices_str, selected_raw_article_sents, selected_article_lcs_paths, FLAGS.batch_size, hps, self._vocab) original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # article_withunks = data.show_art_oovs(original_article, self._vocab) # string # abstract_withunks = data.show_abs_oovs(original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string if FLAGS.first_intact and ssi_idx == 0: decoded_words = selected_article_text.strip().split() decoded_output = selected_article_text else: decoded_words, decoded_output, best_hyp = decode_example( self._sess, self._model, self._vocab, batch, example_idx, hps) best_hyps.append(best_hyp) final_decoded_words.extend(decoded_words) final_decoded_outpus += decoded_output if example_idx < 100 or (example_idx >= 2000 and example_idx < 2100): min_matched_tokens = 2 selected_article_sent_tokens = [ util.process_sent(sent) for sent in selected_raw_article_sents ] highlight_summary_sent_tokens = [decoded_words] highlight_ssi_list, lcs_paths_list, highlight_article_lcs_paths_list, highlight_smooth_article_lcs_paths_list = ssi_functions.get_simple_source_indices_list( highlight_summary_sent_tokens, selected_article_sent_tokens, None, 2, min_matched_tokens) highlighted_html = ssi_functions.html_highlight_sents_in_article( highlight_summary_sent_tokens, highlight_ssi_list, selected_article_sent_tokens, lcs_paths_list=lcs_paths_list, article_lcs_paths_list= highlight_smooth_article_lcs_paths_list) highlight_html_total += highlighted_html + '<br>' if FLAGS.attn_vis and example_idx < 200: self.write_for_attnvis( article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, attn_vis_idx ) # write info to .json file for visualization tool attn_vis_idx += 1 if len(final_decoded_words) >= 100: break gt_ssi_list, gt_alp_list = util.replace_empty_ssis( groundtruth_similar_source_indices_list, raw_article_sents, sys_alp_list=groundtruth_article_lcs_paths_list) highlight_html_gt = '<u>Reference Summary</u><br><br>' for ssi_idx, ssi in enumerate(gt_ssi_list): selected_article_lcs_paths = gt_alp_list[ssi_idx] try: ssi, selected_article_lcs_paths = util.make_ssi_chronological( ssi, selected_article_lcs_paths) except: util.print_vars(ssi, example_idx, selected_article_lcs_paths) raise selected_raw_article_sents = util.reorder( raw_article_sents, ssi) if example_idx < 100 or (example_idx >= 2000 and example_idx < 2100): min_matched_tokens = 2 selected_article_sent_tokens = [ util.process_sent(sent) for sent in selected_raw_article_sents ] highlight_summary_sent_tokens = [ groundtruth_summ_sent_tokens[ssi_idx] ] highlight_ssi_list, lcs_paths_list, highlight_article_lcs_paths_list, highlight_smooth_article_lcs_paths_list = ssi_functions.get_simple_source_indices_list( highlight_summary_sent_tokens, selected_article_sent_tokens, None, 2, min_matched_tokens) highlighted_html = ssi_functions.html_highlight_sents_in_article( highlight_summary_sent_tokens, highlight_ssi_list, selected_article_sent_tokens, lcs_paths_list=lcs_paths_list, article_lcs_paths_list= highlight_smooth_article_lcs_paths_list) highlight_html_gt += highlighted_html + '<br>' if example_idx < 100 or (example_idx >= 2000 and example_idx < 2100): self.write_for_human(raw_article_sents, groundtruth_summ_sents, final_decoded_words, example_idx) highlight_html_total = ssi_functions.put_html_in_two_columns( highlight_html_total, highlight_html_gt) ssi_functions.write_highlighted_html(highlight_html_total, self._highlight_dir, example_idx) # if example_idx % 100 == 0: # attn_dir = os.path.join(self._decode_dir, 'attn_vis_data') # attn_selections.process_attn_selections(attn_dir, self._decode_dir, self._vocab) rouge_functions.write_for_rouge( groundtruth_summ_sents, None, example_idx, self._rouge_ref_dir, self._rouge_dec_dir, decoded_words=final_decoded_words, log=False ) # write ref summary and decoded summary to file, to eval with pyrouge later # if FLAGS.attn_vis: # self.write_for_attnvis(article_withunks, abstract_withunks, decoded_words, best_hyp.attn_dists, best_hyp.p_gens, example_idx) # write info to .json file for visualization tool example_idx += 1 # this is how many examples we've decoded logging.info("Decoder has finished reading dataset for single_pass.") logging.info("Output has been saved in %s and %s.", self._rouge_ref_dir, self._rouge_dec_dir) if len(os.listdir(self._rouge_ref_dir)) != 0: if FLAGS.dataset_name == 'xsum': l_param = 100 else: l_param = 100 logging.info("Now starting ROUGE eval...") results_dict = rouge_functions.rouge_eval(self._rouge_ref_dir, self._rouge_dec_dir, l_param=l_param) rouge_functions.rouge_log(results_dict, self._decode_dir)
def decode(self): """Decode examples until data is exhausted (if FLAGS.single_pass) and return, or decode indefinitely, loading latest checkpoint at regular intervals""" t0 = time.time() counter = 0 f = os.path.join(FLAGS.log_root, "output.txt") # print("----------------"+f) outputfile = codecs.open(f, "w", "utf8") output_result = [] list_of_reference = [] while True: batch = self._batcher.next_batch( ) # 1 example repeated across batch if batch is None: # finished decoding dataset in single_pass mode logging.info("eval_finished") outputfile.close() break print(self._batcher.c_index) original_article = batch.original_articles[0] # string original_abstract = batch.original_abstracts[0] # string original_abstract_sents = batch.original_abstracts_sents[ 0] # list of strings article_withunks = data.show_art_oovs(original_article, self._vocab) # string abstract_withunks = data.show_abs_oovs( original_abstract, self._vocab, (batch.art_oovs[0] if FLAGS.pointer_gen else None)) # string # Run beam search to get best Hypothesis result, all_candidate = self.eval_one_batch_with_candidate( self._sess, self._model, self._vocab, batch) #result = self.eval_one_batch(self._sess, self._model, self._vocab, batch) for i, instance in enumerate(result): if i == len(batch.art_oovs): break if i >= batch.real_length: print("eval done with {} isntances".format( len(output_result))) break out_words = data.outputids2words(instance, self._model._vocab_out, batch.art_oovs[i]) if data.STOP_DECODING in out_words: out_words = out_words[:out_words.index(data.STOP_DECODING)] candidates_value = self.get_condidate_predicate( out_words, all_candidate[i], batch.art_oovs[i]) candidates_value = "_||_".join(candidates_value) output_now = " ".join(out_words) output_result.append(output_now) # refer = " ".join(refer) refer = batch.original_abstracts[i].strip() list_of_reference.append([refer]) outputfile.write(batch.original_articles[i] + '\t' + batch.original_abstracts[i] + '\t' + output_now + '\t' + candidates_value + '\n') bleu = matrix.bleu_score(list_of_reference, output_result) acc = matrix.compute_acc(list_of_reference, output_result) print("bleu : {} acc : {}".format(bleu, acc)) return