def _eval_lines(config_file, start_idx, end_idx, max_steps, interval=10000):
print('config_file: {}'.format(config_file))
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
sweeper = Sweeper(os.path.join(project_root, config_file))
eval = []
eval_lines = []
for k in range(sweeper.total_combinations):
eval.append([])
eval_lines.append([])
for idx in range(start_idx, end_idx):
cfg = sweeper.parse(idx)
cfg.data_root = os.path.join(project_root, 'data', 'output')
log_dir = cfg.get_logdir()
log_path = os.path.join(log_dir, 'log')
try:
with open(log_path, "r") as f:
lines = f.readlines()
except FileNotFoundError:
continue
if len(lines) == 0:
continue
# ugly parse based on the log_file format
try:
num_steps = get_max_steps(lines)
if num_steps >= max_steps:
assert idx % sweeper.total_combinations == cfg.param_setting
avg_eval_steps = extract_line(lines, max_steps, interval=interval)
eval[idx % sweeper.total_combinations].append(np.mean(avg_eval_steps[-int(len(avg_eval_steps)/2):]))
except IndexError:
print(idx)
raise
summary = list(map(lambda x: (x[0], np.mean(x[1]), np.std(x[1]), len(x[1])), enumerate(eval)))
summary = [x for x in summary if np.isnan(x[1]) == False]
# new_summary = []
# for s in summary:
# if np.isnan(s[1]) == False:
# new_summary.append(s)
# print(summary[0])
# print(new_summary[0])
# quit()
summary = sorted(summary, key=lambda s: s[1], reverse=True)
for idx, mean, std, num_runs in summary:
print("Param Setting # {:>3d} | Rewards: {:>10.10f} +/- {:>5.2f} ({:>2d} runs) {} | ".format(
idx, mean, std, num_runs, sweeper.param_setting_from_id(idx)))
eval = []
for k in range(sweeper.total_combinations):
eval.append([])
for idx in range(start_idx, end_idx):
cfg = sweeper.parse(idx)
log_dir = cfg.get_logdir()
log_path = os.path.join(log_dir, 'steps_log')
with open(log_path, "r") as f:
lines = f.readlines()
if len(lines) == 0:
continue
# ugly parse based on the log_file format
num_steps = int(lines[-1].split("|")[1].split(":")[1])
episodes = int(lines[-1].split("|")[2].split(":")[1])
if cfg.max_steps == num_steps:
assert idx % sweeper.total_combinations == cfg.param_setting
eval[idx % sweeper.total_combinations].append(episodes)
summary = list(
map(lambda x: (x[0], np.mean(x[1]), np.std(x[1]), len(x[1])),
enumerate(eval)))
summary = sorted(summary, key=lambda s: s[1], reverse=True)
for idx, mean, std, num_runs in summary:
print(
"Param Setting # {:>3d} | Average num of episodes: {:>10.2f} +/- {:>5.2f} ({:>2d} runs) {} | "
.format(idx, mean, std, num_runs,
sweeper.param_setting_from_id(idx)))
