Esempio n. 1
0
def eval_model(opt, skip_valid=False, skip_test=False, valid_datatype=None):
    """
    Run through an evaluation loop.

    :param opt:
        Any non-default options you wish to set.
    :param bool skip_valid:
        If true skips the valid evaluation, and the second return value will be None.
    :param bool skip_test:
        If true skips the test evaluation, and the third return value will be None.
    :param str valid_datatype:
        If custom datatype required for valid, e.g. train:evalmode, specify here

    :return: (stdout, valid_results, test_results)
    :rtype: (str, dict, dict)

    If model_file is not in opt, then this helper will create a temporary directory
    to store the model files, and clean up afterwards. You can keep the directory
    by disabling autocleanup
    """
    import parlai.scripts.eval_model as ems

    parser = ems.setup_args()
    parser.set_params(**opt)
    parser.set_params(log_every_n_secs=10)
Esempio n. 2
0
def eval_model(opt, skip_valid=False, skip_test=False):
    """
    Run through an evaluation loop.

    :param opt:
        Any non-default options you wish to set.
    :param bool skip_valid:
        If true skips the valid evaluation, and the second return value will be None.
    :param bool skip_test:
        If true skips the test evaluation, and the third return value will be None.

    :return: (stdout, valid_results, test_results)
    :rtype: (str, dict, dict)

    If model_file is not in opt, then this helper will create a temporary directory
    to store the model files, and clean up afterwards. You can keep the directory
    by disabling autocleanup
    """
    import parlai.scripts.eval_model as ems

    parser = ems.setup_args()
    parser.set_params(**opt)
    parser.set_params(log_every_n_secs=10)
    popt = parser.parse_args(print_args=False)

    if popt.get('model_file') and not popt.get('dict_file'):
        popt['dict_file'] = popt['model_file'] + '.dict'

    with capture_output() as output:
        popt['datatype'] = 'valid'
        valid = None if skip_valid else ems.eval_model(popt)
        popt['datatype'] = 'test'
        test = None if skip_test else ems.eval_model(popt)

    return (output.getvalue(), valid, test)
Esempio n. 3
0
    def test_metrics_select(self):
        """
        Test output of running eval_model.
        """
        parser = setup_args()
        parser.set_defaults(
            task='integration_tests',
            model='repeat_label',
            datatype='valid',
            num_examples=5,
            display_examples=False,
            metrics='accuracy,rouge',
        )

        opt = parser.parse_args([], print_args=False)
        valid, test = testing_utils.eval_model(opt)

        self.assertEqual(valid['accuracy'], 1)
        self.assertEqual(valid['rouge-L'], 1)
        self.assertEqual(valid['rouge-1'], 1)
        self.assertEqual(valid['rouge-2'], 1)
        self.assertEqual(test['accuracy'], 1)
        self.assertEqual(test['rouge-L'], 1)
        self.assertEqual(test['rouge-1'], 1)
        self.assertEqual(test['rouge-2'], 1)

        self.assertNotIn('bleu-4', valid)
        self.assertNotIn('bleu-4', test)
Esempio n. 4
0
def eval_model(opt):
    """
    Runs through an evaluation loop.

    :return: (stdout, stderr, valid_results, test_results)
    :rtype: (str, str, dict, dict)

    If model_file is not in opt, then this helper will create a temporary directory
    to store the model files, and clean up afterwards. You can keep the directory
    by disabling autocleanup
    """

    import parlai.scripts.eval_model as ems
    parser = ems.setup_args()
    parser.set_params(**opt)
    popt = parser.parse_args(print_args=False)

    if popt.get('model_file') and not popt.get('dict_file'):
        popt['dict_file'] = popt['model_file'] + '.dict'

    with capture_output() as output:
        popt['datatype'] = 'valid'
        valid = ems.eval_model(popt)
        popt['datatype'] = 'test'
        test = ems.eval_model(popt)

    return (
        output.getvalue(),
        valid,
        test,
    )
Esempio n. 5
0
def main():
    parser = eval_model.setup_args()
    parser.add_distributed_training_args()
    parser.add_argument('--port', type=int, default=61337, help='TCP port number')
    opt = parser.parse_args(print_args=(os.environ['SLURM_PROCID'] == '0'))

    with distributed_utils.slurm_distributed_context(opt) as opt:
        return eval_model.eval_model(opt)
Esempio n. 6
0
def setup_args():
    parser = eval_model.setup_args()
    parser.add_distributed_training_args()
    parser.add_argument('--port',
                        type=int,
                        default=61337,
                        help='TCP port number')
    return parser
Esempio n. 7
0
 def setup_args(cls):
     parser = setup_args()
     parser.add_argument(
         '--search-server',
         type=str,
         default=None,
         help='search server argument for retrieving documents',
     )
     parser.add_argument(
         '--save-dir', type=str, default=None, help='Path to directory to save data'
     )
     parser.set_params(model='projects.seeker.scripts.generate_lm_data:OverlapAgent')
     return parser
Esempio n. 8
0
    def test_save_report(self):
        """
        Test that we can save report from eval model.
        """
        with testing_utils.tempdir() as tmpdir:
            save_report = os.path.join(tmpdir, 'report')
            parser = setup_args()
            parser.set_defaults(
                task='integration_tests',
                model='repeat_label',
                datatype='valid',
                num_examples=5,
                display_examples=False,
                save_world_logs=True,
                report_filename=save_report,
            )

            opt = parser.parse_args([], print_args=False)
            valid, test = testing_utils.eval_model(opt)
Esempio n. 9
0
def main():
    parser = setup_args()
    parser.add_argument('--inference_input_file', type=str, required=True)
    parser.add_argument('--inference_batchsize', type=int, default=1)
    parser.add_argument('--inference_output_file', type=str, default='')
    parser.add_argument('--suffix', type=str, default='predictions')
    opt = parser.parse_args()

    batch_size = opt['inference_batchsize']
    input_context_file = opt['inference_input_file']
    if opt['inference_output_file'] == '':
        assert opt[
            'suffix'] != '', "You have not specify the <inference_output_file>, so the <suffix> must be given!"
        output_file = "{}.{}".format(input_context_file, opt['suffix'])
    else:
        output_dir = os.path.dirname(opt['inference_output_file'])
        if not os.path.isdir(output_dir):
            os.makedirs(output_dir)
        output_file = opt['inference_output_file']
    batch_arr = create_batch_from_file(input_context_file,
                                       batch_size=batch_size)

    inference(opt, batch_arr, output_file)
Esempio n. 10
0
# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
"""Evaluate a pre-trained retriever-reader model on open squad.
"""
from parlai.scripts.eval_model import setup_args, eval_model

if __name__ == '__main__':
    parser = setup_args()
    parser.set_params(
        task='squad:opensquad',
        model='retriever_reader',
        retriever_model_file='models:wikipedia_full/tfidf_retriever/model',
        reader_model_file='models:drqa/squad/model',
    )
    opt = parser.parse_args(print_args=False)
    eval_model(opt, print_parser=parser)
Esempio n. 11
0
def setup_args():
    parser = eval_model.setup_args()
    parser.add_distributed_training_args()
    parser.set_defaults(distributed_world_size=torch.cuda.device_count())
    return parser