def _check_bleu_script(self): """ Checks the correctness of the multi-bleu script. Returns: True/False Raises: OSError: if multi-bleu script not exists, or if evaluation labels file not exits, or if BLEU score is not correct. """ if not gfile.Exists(self._multibleu_script): raise OSError( "File not found. Fail to open multi-bleu scrip: {}".format( self._multibleu_script)) if gfile.Exists(self._eval_labels_file): pseudo_predictions = self._eval_labels_file else: pseudo_predictions = self._eval_labels_file + "0" if not gfile.Exists(pseudo_predictions): raise OSError( "File not found. Fail to open eval_labels_file: {} or {}". format(self._eval_labels_file, pseudo_predictions)) score = multi_bleu_score(self._multibleu_script, self._eval_labels_file, pseudo_predictions) if int(score) < 100: raise OSError( "Fail to run multi-bleu scrip: {}. " "The evaluation output is {} which should be 100".format( self._multibleu_script, score))
def run(self): """ Runs ensemble model. """ self._vocab_source = Vocab( filename=self._model_configs["infer"]["source_words_vocabulary"], bpe_codes_file=self._model_configs["infer"]["source_bpecodes"]) self._vocab_target = Vocab( filename=self._model_configs["infer"]["target_words_vocabulary"], bpe_codes_file=self._model_configs["infer"]["target_bpecodes"]) # build dataset dataset = Dataset(self._vocab_source, self._vocab_target, eval_features_file=[ p["features_file"] for p in self._model_configs["infer_data"] ]) estimator_spec = model_fn_ensemble( self._model_dirs, dataset, weight_scheme=self._weight_scheme, inference_options=self._model_configs["infer"]) predict_op = estimator_spec.predictions sess = self._build_default_session() text_inputter = TextLineInputter( dataset=dataset, data_field_name="eval_features_file", batch_size=self._model_configs["infer"]["batch_size"], maximum_line_length=None) sess.run(tf.global_variables_initializer()) tf.logging.info("Start inference.") overall_start_time = time.time() for feeding_data, param in zip(text_inputter.make_feeding_data(), self._model_configs["infer_data"]): tf.logging.info("Infer Source Features File: {}.".format( param["features_file"])) start_time = time.time() infer(sess=sess, prediction_op=predict_op, feeding_data=feeding_data, output=param["output_file"], vocab_target=self._vocab_target, alpha=self._model_configs["infer"]["length_penalty"], delimiter=self._model_configs["infer"]["delimiter"], output_attention=False, tokenize_output=self._model_configs["infer"]["char_level"], tokenize_script=self._model_configs["infer"] ["tokenize_script"], verbose=True) tf.logging.info("FINISHED {}. Elapsed Time: {}.".format( param["features_file"], str(time.time() - start_time))) if param["labels_file"] is not None: bleu_score = multi_bleu_score( self._model_configs["infer"]["multibleu_script"], param["labels_file"], param["output_file"]) tf.logging.info("BLEU score ({}): {}".format( param["features_file"], bleu_score)) tf.logging.info("Total Elapsed Time: %s" % str(time.time() - overall_start_time))
def _do_evaluation(self, run_context, global_step): """ Infers the evaluation data and computes the BLEU score. Args: run_context: A `SessionRunContext` object. global_step: A python integer, the current training step. """ start_time = time.time() output_prediction_file = self._tmp_trans_file_prefix + str(global_step) sources, hypothesis = infer(sess=run_context.session, prediction_op=self._predict_ops, infer_data=self._infer_data, output=output_prediction_file, vocab_target=self._dataset.vocab_target, vocab_source=self._dataset.vocab_source, delimiter=self._delimiter, output_attention=False, tokenize_output=self._char_level, verbose=False) # print translation samples random_start = random.randint(0, len(hypothesis) - 5) for idx in range(5): tf.logging.info("Sample%d Source: %s" % (idx, self._sources[idx + random_start].strip())) tf.logging.info("Sample%d Encoded Input: %s" % (idx, sources[idx + random_start])) tf.logging.info( "Sample%d Reference: %s" % (idx, self._references[idx + random_start][0].strip())) tf.logging.info("Sample%d Hypothesis: %s\n" % (idx, hypothesis[idx + random_start].strip())) # evaluate with BLEU bleu = multi_bleu_score(hypothesis, self._references) if self._summary_writer is not None: self._summary_writer.add_summary("Metrics/BLEU", bleu, global_step) _, elapsed_time_all = self._timer.update_last_triggered_step( global_step) self._update_bleu_ckpt(run_context, bleu, global_step) tf.logging.info( "Evaluating DEVSET: BLEU=%.2f (Best %.2f) GlobalStep=%d BadCount=%d " "UD %.2f UDfromStart %.2f" % (bleu, self._best_bleu_score, global_step, self._bad_count, time.time() - start_time, elapsed_time_all))
def _do_evaluation(self, run_context, global_step): """ Infers the evaluation data and computes the BLEU score. Args: run_context: A `SessionRunContext` object. global_step: A python integer, the current training step. """ start_time = time.time() output_prediction_file = self._tmp_trans_file_prefix + str(global_step) samples_src, samples_trg = infer( sess=run_context.session, prediction_op=self._predict_ops, feeding_data=self._eval_feeding_data, output=output_prediction_file, vocab_target=self._dataset.vocab_target, alpha=self._model_configs["model_params"] ["inference.length_penalty"], delimiter=self._delimiter, output_attention=False, tokenize_output=self._char_level, tokenize_script=self._tokenize_script, verbose=False) # print translation samples for idx, (s, p) in enumerate(zip(samples_src, samples_trg)): tf.logging.info("Sample%d Source: %s" % (idx, s)) tf.logging.info("Sample%d Prediction: %s\n" % (idx, p)) # evaluate with BLEU bleu = multi_bleu_score(self._multibleu_script, self._eval_labels_file, output_prediction_file) if self._summary_writer is not None: self._summary_writer.add_summary("Metrics/BLEU", bleu, global_step) _, elapsed_time_all = self._timer.update_last_triggered_step( global_step) tf.logging.info( "Evaluating DEVSET: BLEU=%.2f (Best %.2f) GlobalStep=%d UD %.2f UDfromStart %.2f" % (bleu, self._best_bleu_score, global_step, time.time() - start_time, elapsed_time_all)) self._update_bleu_ckpt(run_context, bleu, global_step)
def _do_evaluation(self, run_context, global_step): """ Infers the evaluation data and computes the BLEU score. Args: run_context: A `SessionRunContext` object. global_step: A python integer, the current training step. """ start_time = time.time() output_prediction_file = self._tmp_trans_file_prefix + str(global_step) sources, hypothesis = infer( sess=run_context.session, prediction_op=self._predict_ops, infer_data=self._infer_data, output=output_prediction_file, vocab_target=self._dataset.vocab_target, vocab_source=self._dataset.vocab_source, delimiter=self._delimiter, output_attention=False, tokenize_output=self._char_level, verbose=False) # print translation samples random_start = random.randint(0, len(hypothesis) - 5) for idx in range(5): tf.logging.info("Sample%d Source: %s" % (idx, self._sources[idx + random_start].strip())) tf.logging.info("Sample%d Encoded Input: %s" % (idx, sources[idx + random_start])) tf.logging.info("Sample%d Reference: %s" % (idx, self._references[idx + random_start][0].strip())) tf.logging.info("Sample%d Hypothesis: %s\n" % (idx, hypothesis[idx + random_start].strip())) # evaluate with BLEU bleu = multi_bleu_score(hypothesis, self._references) if self._summary_writer is not None: self._summary_writer.add_summary("Metrics/BLEU", bleu, global_step) _, elapsed_time_all = self._timer.update_last_triggered_step(global_step) self._update_bleu_ckpt(run_context, bleu, global_step) tf.logging.info( "Evaluating DEVSET: BLEU=%.2f (Best %.2f) GlobalStep=%d BadCount=%d " "UD %.2f UDfromStart %.2f" % ( bleu, self._best_bleu_score, global_step, self._bad_count, time.time() - start_time, elapsed_time_all))
def run(self): """Infers data files. """ # build datasets self._vocab_source = Vocab( filename=self._model_configs["infer"]["source_words_vocabulary"], bpe_codes_file=self._model_configs["infer"]["source_bpecodes"], reverse_seq=False) self._vocab_target = Vocab( filename=self._model_configs["infer"]["target_words_vocabulary"], bpe_codes_file=self._model_configs["infer"]["target_bpecodes"], reverse_seq=self._model_configs["train"]["reverse_target"]) # build dataset dataset = Dataset(self._vocab_source, self._vocab_target, eval_features_file=[ p["features_file"] for p in self._model_configs["infer_data"] ]) self._model_configs = update_infer_params( self._model_configs, beam_size=self._model_configs["infer"]["beam_size"], maximum_labels_length=self._model_configs["infer"] ["maximum_labels_length"], length_penalty=self._model_configs["infer"]["length_penalty"]) # build model estimator_spec = model_fn(model_configs=self._model_configs, mode=tf.contrib.learn.ModeKeys.INFER, dataset=dataset) predict_op = estimator_spec.predictions sess = self._build_default_session() text_inputter = TextLineInputter( dataset=dataset, data_field_name="eval_features_file", batch_size=self._model_configs["infer"]["batch_size"]) # reload checkpoint_path = tf.train.latest_checkpoint( self._model_configs["model_dir"]) if checkpoint_path: tf.logging.info("reloading models...") optimistic_restore(sess, checkpoint_path) else: raise OSError( "File NOT Found. Fail to find checkpoint file from: {}".format( self._model_configs["model_dir"])) tf.logging.info("Start inference.") overall_start_time = time.time() for feeding_data, param in zip(text_inputter.make_feeding_data(), self._model_configs["infer_data"]): tf.logging.info("Infer Source Features File: {}.".format( param["features_file"])) start_time = time.time() infer(sess=sess, prediction_op=predict_op, feeding_data=feeding_data, output=param["output_file"], vocab_target=self._vocab_target, alpha=self._model_configs["infer"]["length_penalty"], delimiter=self._model_configs["infer"]["delimiter"], output_attention=param["output_attention"], tokenize_output=self._model_configs["infer"]["char_level"], tokenize_script=self._model_configs["infer"] ["tokenize_script"], verbose=True) tf.logging.info("FINISHED {}. Elapsed Time: {}.".format( param["features_file"], str(time.time() - start_time))) if param["labels_file"] is not None: bleu_score = multi_bleu_score( self._model_configs["infer"]["multibleu_script"], param["labels_file"], param["output_file"]) tf.logging.info("BLEU score ({}): {}".format( param["features_file"], bleu_score)) tf.logging.info("Total Elapsed Time: %s" % str(time.time() - overall_start_time))