testset,
cpu_pool=cpu_pool,
tokenizer=tokenizer,
oracle=oracle_network,
guesser=guesser_network,
qgen=qgen_network,
batch_size=batch_size * 2,
logger=logger)
logger.info(">>> New Objects <<<")
compute_qgen_accuracy(sess,
trainset,
batchifier=train_batchifier,
evaluator=looper_evaluator,
tokenizer=tokenizer,
mode=mode_to_evaluate,
save_path=save_path,
cpu_pool=cpu_pool,
batch_size=batch_size,
store_games=args.store_games,
dump_suffix="init.new_object")
logger.info(">>> New Games <<<")
compute_qgen_accuracy(sess,
testset,
batchifier=eval_batchifier,
evaluator=looper_evaluator,
tokenizer=tokenizer,
mode=mode_to_evaluate,
save_path=save_path,
cpu_pool=cpu_pool,
loop_config = {} # fake config
loop_config['loop'] = {}
loop_config['loop']['max_question'] = 5
game_engine = BasicLooper(loop_config,
oracle_wrapper=oracle_wrapper,
guesser_wrapper=guesser_wrapper,
qgen_wrapper=qgen_wrapper,
tokenizer=tokenizer,
batch_size=64)
logger.info(">>> New Objects <<<")
compute_qgen_accuracy(sess_loop,
trainset,
batchifier=train_batchifier,
looper=game_engine,
mode=mode_to_evaluate,
cpu_pool=cpu_pool,
batch_size=batch_size,
name="ini.new_object",
save_path=xp_manager.dir_xp,
store_games=True)
logger.info(">>> New Games <<<")
compute_qgen_accuracy(sess_loop,
testset,
batchifier=eval_batchifier,
looper=game_engine,
mode=mode_to_evaluate,
cpu_pool=cpu_pool,
batch_size=batch_size,
name="ini.new_images",
save_path=xp_manager.dir_xp,
oracle_wrapper=oracle_wrapper,
guesser_wrapper=guesser_wrapper,
qgen_wrapper=qgen_wrapper,
tokenizer=tokenizer,
batch_size=loop_config["optimizer"]["batch_size"])
# Compute the initial scores
logger.info(">>>-------------- INITIAL SCORE ---------------------<<<")
for split in ["nd_test", "nd_valid", "od_test", "od_valid"]:
logger.info("Loading dataset split {}".format(split))
testset = Dataset(args.data_dir, split, "guesswhat_nocaps",
image_builder, crop_builder)
logger.info(">>> New Games <<<")
dump_suffix = "gameplay_{}_{}".format(
split, "rl" if args.rl_identifier else "sl")
compute_qgen_accuracy(sess,
testset,
batchifier=eval_batchifier,
evaluator=looper_evaluator,
tokenizer=tokenizer,
mode=mode_to_evaluate,
save_path=save_path,
cpu_pool=cpu_pool,
batch_size=batch_size,
store_games=args.store_games,
dump_suffix=dump_suffix)
logger.info(
">>>------------------------------------------------<<<")