def eval_model(opt, skip_valid=False, skip_test=False, valid_datatype=None):
"""
Run through an evaluation loop.
:param opt:
Any non-default options you wish to set.
:param bool skip_valid:
If true skips the valid evaluation, and the second return value will be None.
:param bool skip_test:
If true skips the test evaluation, and the third return value will be None.
:param str valid_datatype:
If custom datatype required for valid, e.g. train:evalmode, specify here
:return: (stdout, valid_results, test_results)
:rtype: (str, dict, dict)
If model_file is not in opt, then this helper will create a temporary directory
to store the model files, and clean up afterwards. You can keep the directory
by disabling autocleanup
"""
import parlai.scripts.eval_model as ems
parser = ems.setup_args()
parser.set_params(**opt)
parser.set_params(log_every_n_secs=10)
def eval_model(opt, skip_valid=False, skip_test=False):
"""
Run through an evaluation loop.
:param opt:
Any non-default options you wish to set.
:param bool skip_valid:
If true skips the valid evaluation, and the second return value will be None.
:param bool skip_test:
If true skips the test evaluation, and the third return value will be None.
:return: (stdout, valid_results, test_results)
:rtype: (str, dict, dict)
If model_file is not in opt, then this helper will create a temporary directory
to store the model files, and clean up afterwards. You can keep the directory
by disabling autocleanup
"""
import parlai.scripts.eval_model as ems
parser = ems.setup_args()
parser.set_params(**opt)
parser.set_params(log_every_n_secs=10)
popt = parser.parse_args(print_args=False)
if popt.get('model_file') and not popt.get('dict_file'):
popt['dict_file'] = popt['model_file'] + '.dict'
with capture_output() as output:
popt['datatype'] = 'valid'
valid = None if skip_valid else ems.eval_model(popt)
popt['datatype'] = 'test'
test = None if skip_test else ems.eval_model(popt)
return (output.getvalue(), valid, test)
def test_metrics_select(self):
"""
Test output of running eval_model.
"""
parser = setup_args()
parser.set_defaults(
task='integration_tests',
model='repeat_label',
datatype='valid',
num_examples=5,
display_examples=False,
metrics='accuracy,rouge',
)
opt = parser.parse_args([], print_args=False)
valid, test = testing_utils.eval_model(opt)
self.assertEqual(valid['accuracy'], 1)
self.assertEqual(valid['rouge-L'], 1)
self.assertEqual(valid['rouge-1'], 1)
self.assertEqual(valid['rouge-2'], 1)
self.assertEqual(test['accuracy'], 1)
self.assertEqual(test['rouge-L'], 1)
self.assertEqual(test['rouge-1'], 1)
self.assertEqual(test['rouge-2'], 1)
self.assertNotIn('bleu-4', valid)
self.assertNotIn('bleu-4', test)
def eval_model(opt):
"""
Runs through an evaluation loop.
:return: (stdout, stderr, valid_results, test_results)
:rtype: (str, str, dict, dict)
If model_file is not in opt, then this helper will create a temporary directory
to store the model files, and clean up afterwards. You can keep the directory
by disabling autocleanup
"""
import parlai.scripts.eval_model as ems
parser = ems.setup_args()
parser.set_params(**opt)
popt = parser.parse_args(print_args=False)
if popt.get('model_file') and not popt.get('dict_file'):
popt['dict_file'] = popt['model_file'] + '.dict'
with capture_output() as output:
popt['datatype'] = 'valid'
valid = ems.eval_model(popt)
popt['datatype'] = 'test'
test = ems.eval_model(popt)
return (
output.getvalue(),
valid,
test,
)
def main():
parser = eval_model.setup_args()
parser.add_distributed_training_args()
parser.add_argument('--port', type=int, default=61337, help='TCP port number')
opt = parser.parse_args(print_args=(os.environ['SLURM_PROCID'] == '0'))
with distributed_utils.slurm_distributed_context(opt) as opt:
return eval_model.eval_model(opt)
def setup_args():
parser = eval_model.setup_args()
parser.add_distributed_training_args()
parser.add_argument('--port',
type=int,
default=61337,
help='TCP port number')
return parser
def setup_args(cls):
parser = setup_args()
parser.add_argument(
'--search-server',
type=str,
default=None,
help='search server argument for retrieving documents',
)
parser.add_argument(
'--save-dir', type=str, default=None, help='Path to directory to save data'
)
parser.set_params(model='projects.seeker.scripts.generate_lm_data:OverlapAgent')
return parser
def test_save_report(self):
"""
Test that we can save report from eval model.
"""
with testing_utils.tempdir() as tmpdir:
save_report = os.path.join(tmpdir, 'report')
parser = setup_args()
parser.set_defaults(
task='integration_tests',
model='repeat_label',
datatype='valid',
num_examples=5,
display_examples=False,
save_world_logs=True,
report_filename=save_report,
)
opt = parser.parse_args([], print_args=False)
valid, test = testing_utils.eval_model(opt)
def main():
parser = setup_args()
parser.add_argument('--inference_input_file', type=str, required=True)
parser.add_argument('--inference_batchsize', type=int, default=1)
parser.add_argument('--inference_output_file', type=str, default='')
parser.add_argument('--suffix', type=str, default='predictions')
opt = parser.parse_args()
batch_size = opt['inference_batchsize']
input_context_file = opt['inference_input_file']
if opt['inference_output_file'] == '':
assert opt[
'suffix'] != '', "You have not specify the <inference_output_file>, so the <suffix> must be given!"
output_file = "{}.{}".format(input_context_file, opt['suffix'])
else:
output_dir = os.path.dirname(opt['inference_output_file'])
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
output_file = opt['inference_output_file']
batch_arr = create_batch_from_file(input_context_file,
batch_size=batch_size)
inference(opt, batch_arr, output_file)
# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
"""Evaluate a pre-trained retriever-reader model on open squad.
"""
from parlai.scripts.eval_model import setup_args, eval_model
if __name__ == '__main__':
parser = setup_args()
parser.set_params(
task='squad:opensquad',
model='retriever_reader',
retriever_model_file='models:wikipedia_full/tfidf_retriever/model',
reader_model_file='models:drqa/squad/model',
)
opt = parser.parse_args(print_args=False)
eval_model(opt, print_parser=parser)
def setup_args():
parser = eval_model.setup_args()
parser.add_distributed_training_args()
parser.set_defaults(distributed_world_size=torch.cuda.device_count())
return parser