def _eval_for_set( name, task, interaction_file, prediction_file, global_step, ): """Computes eval metric from predictions.""" if task in [ tasks.Task.SQA, tasks.Task.WTQ, tasks.Task.WIKISQL, tasks.Task.WIKISQL_SUPERVISED ]: if not tf.io.gfile.exists(prediction_file): _warn( f"Can't evaluate for {name} because {prediction_file} doesn't exist.") return test_examples = calc_metrics_utils.read_data_examples_from_interactions( interaction_file) calc_metrics_utils.read_predictions( predictions_path=prediction_file, examples=test_examples, ) denotation_accuracy = calc_metrics_utils.calc_denotation_accuracy( examples=test_examples, denotation_errors_path=None, predictions_file_name=None, ) _print(f'{name} denotation accuracy: {denotation_accuracy:0.4f}') else: raise ValueError(f'Unknown task: {task.name}')
def _eval_for_set( model_dir, name, task, interaction_file, prediction_file, global_step, ): """Computes eval metric from predictions.""" if not tf.io.gfile.exists(prediction_file): _warn( f"Can't evaluate for {name} because {prediction_file} doesn't exist." ) return test_examples = calc_metrics_utils.read_data_examples_from_interactions( interaction_file) calc_metrics_utils.read_predictions( predictions_path=prediction_file, examples=test_examples, ) if task in [ tasks.Task.SQA, tasks.Task.WTQ, tasks.Task.WIKISQL, tasks.Task.WIKISQL_SUPERVISED ]: denotation_accuracy = calc_metrics_utils.calc_denotation_accuracy( examples=test_examples, denotation_errors_path=None, predictions_file_name=None, ) if global_step is not None: _create_measurements_for_metrics( {'denotation_accuracy': denotation_accuracy}, global_step=global_step, model_dir=model_dir, name=name, ) elif task == tasks.Task.TABFACT: accuracy = calc_metrics_utils.calc_classification_accuracy( test_examples) if global_step is not None: _create_measurements_for_metrics( {'accuracy': accuracy}, global_step=global_step, model_dir=model_dir, name=name, ) else: raise ValueError(f'Unknown task: {task.name}')
def main(_): examples = calc_metrics_utils.read_data_examples_from_interactions( FLAGS.interactions_file) prediction_file_name = os.path.basename(FLAGS.prediction_files) calc_metrics_utils.read_predictions(FLAGS.prediction_files, examples) if FLAGS.is_strong_supervision_available: results = calc_metrics_utils.calc_structure_metrics( examples, FLAGS.denotation_errors_path) print('%s: joint_accuracy=%s' % (FLAGS.prediction_files, results.joint_acc)) denotation_accuracy = calc_metrics_utils.calc_denotation_accuracy( examples, FLAGS.denotation_errors_path, prediction_file_name) print('%s: denotation_accuracy=%s' % (FLAGS.prediction_files, denotation_accuracy))