def run_avg_results():
"""
Useful for generating average results for more experiments (logs).
"""
# List of logs to be measured (tested)
items = [
"logs_2017-06-23_14-16-00", "logs_2017-06-23_14-16-59",
"logs_2017-06-23_14-17-58", "logs_2017-06-23_14-18-48",
"logs_2017-06-23_14-19-39"
]
results = []
game = "2048"
evals = 1000
for item in items:
prefix = "C:/Users/Jan/Documents/GitHub/general-ai/Experiments/best_models_repeats/2048/MLP+ES/"
postfix = "/best/best_0.json"
file_name = prefix + item + postfix
logdir = prefix + item
# SELECT PROPER MODEL
model = MLP.load_from_file(file_name, game)
# model = EchoState.load_from_file(file_name, game)
# RUN MODEL
# 2048
result = run_2048_extended(model, evals)
# MARIO
# result = eval_mario_winrate(model=model, evals=evals, level="spikes", vis_on=False)
# ALHAMBRA
# First element is result of our model (rest are original models from previous work)
# result = eval_alhambra_avg_score(model, evals)[0]
# TORCS
# For reinforcement learning, please run model separately (tensorflow needs to be restarted)
results.append(result)
results = np.array(results)
file_name = "{}_stats_{}.txt".format(game,
utils.miscellaneous.get_pretty_time())
with open(file_name, "w") as f:
f.write("--GAME {} STATISTICS-- {} trainings of the same model".format(
game.upper(), len(items)))
f.write(os.linesep)
f.write("Model: {}".format(model.get_name()))
f.write(os.linesep)
f.write("Total games: {} (for each model)".format(evals))
f.write(os.linesep)
f.write("MAX TEST: {}".format(np.max(results)))
f.write(os.linesep)
f.write("AVG TEST: {}".format(np.mean(results)))
f.write(os.linesep)
f.write("MIN TEST: {}".format(np.min(results)))
def run_model_evaluator():
"""
Used for evaluating learned models, to benchmark them and get avg results.
For example, to run 1000 games and plot results.
Set file_name for example as "C:/Users/Jan/Documents/GitHub/general-ai/Experiments/ESN+EA/torcs/logs_2017-06-20_19-09-27/best/best_0.json" (the whole path)
for evolutionary based experiments. For deep reinforcement (DQN or DDPG) based techniques use logdir, for example as:
"C:/Users/Jan/Documents/GitHub/general-ai/Experiments/DQN/mario/logs_2017-04-19_16-26-07", where directory stores config files (model settings), model checkpoint etc.
Then feel free to use some of the prepared "test functions". Result (if exists) is written to same directory as this file (e.q. /utils).
"""
np.random.seed(930615)
# Before using game 2048, check it's encoding
game = "2048"
evals = 1000
# SELECT FILE (direct model for evolutionary or directory for reinforcement)
file_name = "C:/Users/Jan/Documents/GitHub/general-ai/Experiments/MLP+ES/2048/logs_2017-02-21_17-24-07/best/best_0.json"
# logdir = "C:/Users/Jan/Documents/GitHub/general-ai/Experiments/DDPG/torcs/logs_2017-04-29_11-39-44"
# SELECT MODEL (trained, based on file selected)
# esn = EchoState.load_from_file(file_name, game)
mlp = MLP.load_from_file(file_name, game)
# random = Random(game)
# ddpg = LearnedDDPG(logdir)
# dqn = LearnedDQN(logdir)
# RUN MODEL TEST
# eval_alhambra_winrate(mlp, evals)
# run_random_model(game, evals)
run_2048_extended(mlp, evals)
# eval_mario_winrate(model=dqn, evals=evals, level="spikes", vis_on=False)
# run_torcs_vis_on(model=ddpg, evals=evals)
# general model comparison (graph of score)
# compare_models(game, evals, ddpg)
"""