def make_arg_parser():
parser = train.make_arg_parser()
parser.add_argument("model_prefix")
parser.add_argument("--beam_size", type=int, default=1)
parser.add_argument("--batch_size", type=int, default=100)
parser.add_argument("--eval_file")
return parser
def make_arg_parser():
# parser = train_speaker.make_arg_parser()
# TODO: hack, this only works because the follower has extra parameters
# that the speaker lacks!
parser = train.make_arg_parser()
parser.add_argument("speaker_model_prefix")
parser.add_argument("pred_results_output_file")
parser.add_argument("--batch_size", type=int, default=20)
parser.add_argument("--pred_splits",
nargs="+",
default=["data_augmentation_paths"])
# for rational self-play generation
parser.add_argument("--follower_model_prefix",
help="generate data from a rational speaker "
"(must also pass --rational_speaker_weights")
parser.add_argument("--rational_speaker_weights",
type=float,
nargs="+",
help="list of speaker weights in range [0.0, 1.0] to "
"use with rational speaker (must also pass "
"follower_model_prefix)")
parser.add_argument("--rational_speaker_n_candidates",
type=int,
default=40)
return parser
def make_arg_parser():
parser = train.make_arg_parser()
parser.add_argument("speaker_prefix")
parser.add_argument("follower_prefix")
parser.add_argument("--beam_size", type=int, default=10)
parser.add_argument("--batch_size", type=int, default=30)
parser.add_argument("--include_gold", action='store_true')
parser.add_argument("--output_file")
parser.add_argument("--mask_undo", action='store_true')
return parser
def make_arg_parser():
parser = train.make_arg_parser()
parser.add_argument("--max_episode_len", type=int, default=40)
parser.add_argument("--gamma", type=float, default=0.21)
parser.add_argument("--mean", action='store_true')
parser.add_argument("--logit", action='store_true')
parser.add_argument("--early_stop", action='store_true')
parser.add_argument("--revisit", action='store_true')
parser.add_argument("--inject_stop", action='store_true')
parser.add_argument("--load_reranker", type=str, default='')
parser.add_argument("--K", type=int, default=10)
parser.add_argument("--load_speaker", type=str,
default='./tasks/R2R/experiments/release/speaker_final_release')
parser.add_argument("--job", choices=['search','sweep','train','cache','test'],default='search')
return parser
def make_arg_parser():
parser = train.make_arg_parser()
parser.add_argument("follower_prefix")
parser.add_argument("speaker_prefix")
parser.add_argument("--include_gold", action='store_true')
parser.add_argument("--output_file")
parser.add_argument("--eval_file")
parser.add_argument("--compute_oracle", action='store_true')
parser.add_argument("--mask_undo", action='store_true')
parser.add_argument("--state_factored_search", action='store_true')
parser.add_argument("--state_first_n_ws_key", type=int, default=4)
parser.add_argument("--physical_traversal", action='store_true')
parser.add_argument("--debug_rational", action='store_true')
return parser
def make_arg_parser():
parser = train.make_arg_parser()
parser.add_argument('--gamma', type=float, default=0.21)
parser.add_argument('--mean', action='store_true')
parser.add_argument('--logit', action='store_true')
parser.add_argument('--early_stop', action='store_true')
parser.add_argument('--revisit', action='store_true')
parser.add_argument('--inject_stop', action='store_true')
parser.add_argument('--load_reranker', type=str, default='')
parser.add_argument('--K', type=int, default=20)
parser.add_argument('--beam', action='store_true')
parser.add_argument('--load_speaker', type=str, default='')
parser.add_argument('--job',
choices=['search', 'sweep', 'train', 'cache', 'test'],
default='search')
return parser
score_summary[metric + '_std'])
for metric in METRICS
])
print('\n'.join([header, numbers]))
else:
header = ','.join(['{}'.format(metric) for metric in METRICS])
numbers = ','.join([
'{:4.3f}'.format(score_summary[metric]) for metric in METRICS
])
print('\n'.join([header, numbers]))
csv_file.write(numbers + '\n')
csv_file.write(json.dumps(score_summary))
csv_file.close()
if __name__ == '__main__':
from train import make_arg_parser
parser = make_arg_parser()
# TODO: take function to run as argument
parser.add_argument('--results_path', type=str, default='')
parser.add_argument('--nfolds', default=1, type=int)
parser.add_argument('--function',
default='eval_outfiles',
choices=['eval_outfiles', 'eval_simple_agents'])
functions = {
'eval_outfiles': eval_outfiles,
'eval_simple_agents': eval_simple_agents
}
utils.run(parser, None, functions=functions)
outfiles = [
train.RESULT_DIR + 'seq2seq_teacher_imagenet_%s_iter_5000.json',
train.RESULT_DIR + 'seq2seq_sample_imagenet_%s_iter_20000.json'
]
for outfile in outfiles:
for split in ['val_seen', 'val_unseen']:
ev = Evaluation([split])
score_summary, _ = ev.score_file(outfile % split)
print('\n%s' % outfile)
pp.pprint(score_summary)
def eval_outfiles(outfolder):
splits = ['val_seen', 'val_unseen']
for _f in os.listdir(outfolder):
outfile = os.path.join(outfolder, _f)
_splits = []
for s in splits:
if s in outfile:
_splits.append(s)
ev = Evaluation(_splits)
score_summary, _ = ev.score_file(outfile)
print('\n', outfile)
pp.pprint(score_summary)
if __name__ == '__main__':
from train import make_arg_parser
utils.run(make_arg_parser(), eval_simple_agents)
# eval_seq2seq()
