def make_arg_parser(): parser = train.make_arg_parser() parser.add_argument("model_prefix") parser.add_argument("--beam_size", type=int, default=1) parser.add_argument("--batch_size", type=int, default=100) parser.add_argument("--eval_file") return parser
def make_arg_parser(): # parser = train_speaker.make_arg_parser() # TODO: hack, this only works because the follower has extra parameters # that the speaker lacks! parser = train.make_arg_parser() parser.add_argument("speaker_model_prefix") parser.add_argument("pred_results_output_file") parser.add_argument("--batch_size", type=int, default=20) parser.add_argument("--pred_splits", nargs="+", default=["data_augmentation_paths"]) # for rational self-play generation parser.add_argument("--follower_model_prefix", help="generate data from a rational speaker " "(must also pass --rational_speaker_weights") parser.add_argument("--rational_speaker_weights", type=float, nargs="+", help="list of speaker weights in range [0.0, 1.0] to " "use with rational speaker (must also pass " "follower_model_prefix)") parser.add_argument("--rational_speaker_n_candidates", type=int, default=40) return parser
def make_arg_parser(): parser = train.make_arg_parser() parser.add_argument("speaker_prefix") parser.add_argument("follower_prefix") parser.add_argument("--beam_size", type=int, default=10) parser.add_argument("--batch_size", type=int, default=30) parser.add_argument("--include_gold", action='store_true') parser.add_argument("--output_file") parser.add_argument("--mask_undo", action='store_true') return parser
def make_arg_parser(): parser = train.make_arg_parser() parser.add_argument("--max_episode_len", type=int, default=40) parser.add_argument("--gamma", type=float, default=0.21) parser.add_argument("--mean", action='store_true') parser.add_argument("--logit", action='store_true') parser.add_argument("--early_stop", action='store_true') parser.add_argument("--revisit", action='store_true') parser.add_argument("--inject_stop", action='store_true') parser.add_argument("--load_reranker", type=str, default='') parser.add_argument("--K", type=int, default=10) parser.add_argument("--load_speaker", type=str, default='./tasks/R2R/experiments/release/speaker_final_release') parser.add_argument("--job", choices=['search','sweep','train','cache','test'],default='search') return parser
def make_arg_parser(): parser = train.make_arg_parser() parser.add_argument("follower_prefix") parser.add_argument("speaker_prefix") parser.add_argument("--include_gold", action='store_true') parser.add_argument("--output_file") parser.add_argument("--eval_file") parser.add_argument("--compute_oracle", action='store_true') parser.add_argument("--mask_undo", action='store_true') parser.add_argument("--state_factored_search", action='store_true') parser.add_argument("--state_first_n_ws_key", type=int, default=4) parser.add_argument("--physical_traversal", action='store_true') parser.add_argument("--debug_rational", action='store_true') return parser
def make_arg_parser(): parser = train.make_arg_parser() parser.add_argument('--gamma', type=float, default=0.21) parser.add_argument('--mean', action='store_true') parser.add_argument('--logit', action='store_true') parser.add_argument('--early_stop', action='store_true') parser.add_argument('--revisit', action='store_true') parser.add_argument('--inject_stop', action='store_true') parser.add_argument('--load_reranker', type=str, default='') parser.add_argument('--K', type=int, default=20) parser.add_argument('--beam', action='store_true') parser.add_argument('--load_speaker', type=str, default='') parser.add_argument('--job', choices=['search', 'sweep', 'train', 'cache', 'test'], default='search') return parser
score_summary[metric + '_std']) for metric in METRICS ]) print('\n'.join([header, numbers])) else: header = ','.join(['{}'.format(metric) for metric in METRICS]) numbers = ','.join([ '{:4.3f}'.format(score_summary[metric]) for metric in METRICS ]) print('\n'.join([header, numbers])) csv_file.write(numbers + '\n') csv_file.write(json.dumps(score_summary)) csv_file.close() if __name__ == '__main__': from train import make_arg_parser parser = make_arg_parser() # TODO: take function to run as argument parser.add_argument('--results_path', type=str, default='') parser.add_argument('--nfolds', default=1, type=int) parser.add_argument('--function', default='eval_outfiles', choices=['eval_outfiles', 'eval_simple_agents']) functions = { 'eval_outfiles': eval_outfiles, 'eval_simple_agents': eval_simple_agents } utils.run(parser, None, functions=functions)
outfiles = [ train.RESULT_DIR + 'seq2seq_teacher_imagenet_%s_iter_5000.json', train.RESULT_DIR + 'seq2seq_sample_imagenet_%s_iter_20000.json' ] for outfile in outfiles: for split in ['val_seen', 'val_unseen']: ev = Evaluation([split]) score_summary, _ = ev.score_file(outfile % split) print('\n%s' % outfile) pp.pprint(score_summary) def eval_outfiles(outfolder): splits = ['val_seen', 'val_unseen'] for _f in os.listdir(outfolder): outfile = os.path.join(outfolder, _f) _splits = [] for s in splits: if s in outfile: _splits.append(s) ev = Evaluation(_splits) score_summary, _ = ev.score_file(outfile) print('\n', outfile) pp.pprint(score_summary) if __name__ == '__main__': from train import make_arg_parser utils.run(make_arg_parser(), eval_simple_agents) # eval_seq2seq()