def test_output(self):
"""Test output of running eval_model"""
class display_output(object):
def __init__(self):
self.data = []
def write(self, s):
self.data.append(s)
def __str__(self):
return "".join(self.data)
old_out = sys.stdout
output = display_output()
try:
sys.stdout = output
eval_model(self.opt, self.parser, printargs=False)
finally:
# restore sys.stdout
sys.stdout = old_out
str_output = str(output)
self.assertTrue(len(str_output) > 0, "Output is empty")
# decode the output
scores = str_output.split("\n---\n")
for i in range(1, len(scores)):
score = ast.literal_eval(scores[i])
# check totals
self.assertTrue(score['total'] == i,
"Total is incorrect")
# accuracy should be one
self.assertTrue(score['accuracy'] == 1,
"accuracy != 1")
def test_output(self):
"""Test output of running eval_model"""
class display_output(object):
def __init__(self):
self.data = []
def write(self, s):
self.data.append(s)
def __str__(self):
return "".join(self.data)
old_out = sys.stdout
output = display_output()
try:
sys.stdout = output
eval_model(self.opt, self.parser, printargs=False)
finally:
# restore sys.stdout
sys.stdout = old_out
str_output = str(output)
self.assertTrue(len(str_output) > 0, "Output is empty")
# decode the output
scores = str_output.split("\n---\n")
for i in range(1, len(scores)):
score = ast.literal_eval(scores[i])
# check totals
self.assertTrue(score['total'] == i, "Total is incorrect")
# accuracy should be one
self.assertTrue(score['accuracy'] == 1, "accuracy != 1")
def test_hogwild_eval(self):
"""Test eval with numthreads > 1 and batchsize in [1,2,3]."""
parser = setup_args()
NUM_EXS = 500
parser.set_defaults(
task='tasks.repeat:RepeatTeacher:{}'.format(NUM_EXS),
model='repeat_label',
datatype='valid',
num_examples=-1,
display_examples=False,
)
old_out = sys.stdout
output = display_output()
try:
sys.stdout = output
for nt in [2, 5, 10]:
parser.set_defaults(numthreads=nt)
for bs in [1, 2, 3]:
parser.set_defaults(batchsize=bs)
parser.set_defaults(batch_sort=(bs % 2 == 0))
report = eval_model(parser, printargs=False)
self.assertEqual(report['total'], NUM_EXS)
finally:
# restore sys.stdout
sys.stdout = old_out
def test_output(self):
"""Test output of running eval_model"""
class display_output(object):
def __init__(self):
self.data = []
def write(self, s):
self.data.append(s)
def __str__(self):
return "".join(self.data)
parser = setup_args()
parser.set_defaults(
task='tasks.repeat:RepeatTeacher:10',
model='repeat_label',
datatype='valid',
num_examples=5,
display_examples=False,
)
old_out = sys.stdout
output = display_output()
try:
sys.stdout = output
opt = parser.parse_args(print_args=False)
eval_model(opt, print_parser=parser)
finally:
# restore sys.stdout
sys.stdout = old_out
str_output = str(output)
self.assertTrue(len(str_output) > 0, "Output is empty")
# decode the output
scores = str_output.split("\n---\n")
for i in range(1, len(scores)):
score = ast.literal_eval(scores[i])
# check totals
self.assertTrue(score['total'] == i, "Total is incorrect")
# accuracy should be one
self.assertTrue(score['accuracy'] == 1, "accuracy != 1")
from download_models import build
from parlai.core.params import ParlaiParser
from examples.eval_model import eval_model
'''Evaluate pre-trained model trained for hits@1 metric
Key-Value Memory Net model trained on personachat using persona 'self'
'''
if __name__ == '__main__':
parser = ParlaiParser(add_model_args=True)
parser.add_argument('-n', '--num-examples', default=100000000)
parser.add_argument('-d', '--display-examples', type='bool', default=False)
parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=2)
parser.set_defaults(
task='personachat:self',
model='projects.personachat.kvmemnn.kvmemnn:Kvmemnn',
model_file=
'models:personachat/kvmemnn/kvmemnn/persona-self_rephraseTrn-True_rephraseTst-False_lr-0.1_esz-500_margin-0.1_tfidf-False_shareEmb-True_hops1_lins0_model',
datatype='test',
numthreads=8)
opt = parser.parse_args()
# build all profile memory models
fnames = ['kvmemnn.tgz']
opt['model_type'] = 'kvmemnn' # for builder
build(opt, fnames)
# add additional model args
opt['interactive_mode'] = False
eval_model(parser)
# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
"""Evaluate pre-trained model trained for ppl metric.
This seq2seq model was trained on convai2:self.
"""
from projects.convai2.baselines.download_models import download
from parlai.core.params import ParlaiParser
from examples.eval_model import setup_args, eval_model
from parlai.agents.seq2seq.seq2seq import Seq2seqAgent
if __name__ == '__main__':
parser = setup_args()
parser.set_defaults(
task='convai2:self',
model='seq2seq',
model_file=
'models:convai2/seq2seq/convai2_self_seq2seq_model/convai2_self_seq2seq_model',
dict_file='models:convai2/seq2seq/dict_convai2_self/dict_convai2_self',
datatype='valid',
batchsize=128,
)
opt = parser.parse_args()
download(opt, 'convai2/seq2seq', 'convai2_self_seq2seq_model.tgz')
download(opt, 'convai2/seq2seq', 'dict_convai2_self')
eval_model(parser, printargs=False)
Profile memory model trained on personachat using persona 'self'
Run from ParlAI directory
'''
if __name__ == '__main__':
parser = ParlaiParser(add_model_args=True)
parser.add_argument('-n', '--num-examples', default=100000000)
parser.add_argument('-d', '--display-examples', type='bool', default=False)
parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=2)
parser.set_defaults(
task='personachat:self',
model='projects.personachat.persona_seq2seq:PersonachatSeqseqAgentSplit',
model_file='data/models/personachat/profile_memory/profilememory_mem2_reweight_sharelt_encdropout0.2_selfpersona_useall_attn_general_lstm_1024_1_1e-3_0.1',
datatype='test'
)
PersonachatSeqseqAgentSplit.add_cmdline_args(parser)
opt = parser.parse_args()
opt['model_type'] = 'profile_memory'
# build all profile memory models
fnames = ['profilememory_mem2_reweight_sharelt_encdropout0.2_selfpersona_useall_attn_general_lstm_1024_1_1e-3_0.1',
'profilememory_learnreweight_sharelt_encdropout0.4_s2s_usepersona_self_useall_attn_general_lstm_1024_1_1e-3_0.1',
'fulldict.dict']
build(opt, fnames)
# add additional model args
opt['dict_file'] = 'data/models/personachat/profile_memory/fulldict.dict'
opt['rank_candidates'] = True
eval_model(opt, parser)
