def _eval_lines(config_file, start_idx, end_idx, max_steps, interval=10000): print('config_file: {}'.format(config_file)) project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) sweeper = Sweeper(os.path.join(project_root, config_file)) eval = [] eval_lines = [] for k in range(sweeper.total_combinations): eval.append([]) eval_lines.append([]) for idx in range(start_idx, end_idx): cfg = sweeper.parse(idx) cfg.data_root = os.path.join(project_root, 'data', 'output') log_dir = cfg.get_logdir() log_path = os.path.join(log_dir, 'log') try: with open(log_path, "r") as f: lines = f.readlines() except FileNotFoundError: continue if len(lines) == 0: continue # ugly parse based on the log_file format try: num_steps = get_max_steps(lines) if num_steps >= max_steps: assert idx % sweeper.total_combinations == cfg.param_setting avg_eval_steps = extract_line(lines, max_steps, interval=interval) eval[idx % sweeper.total_combinations].append(np.mean(avg_eval_steps[-int(len(avg_eval_steps)/2):])) except IndexError: print(idx) raise summary = list(map(lambda x: (x[0], np.mean(x[1]), np.std(x[1]), len(x[1])), enumerate(eval))) summary = [x for x in summary if np.isnan(x[1]) == False] # new_summary = [] # for s in summary: # if np.isnan(s[1]) == False: # new_summary.append(s) # print(summary[0]) # print(new_summary[0]) # quit() summary = sorted(summary, key=lambda s: s[1], reverse=True) for idx, mean, std, num_runs in summary: print("Param Setting # {:>3d} | Rewards: {:>10.10f} +/- {:>5.2f} ({:>2d} runs) {} | ".format( idx, mean, std, num_runs, sweeper.param_setting_from_id(idx)))
eval = [] for k in range(sweeper.total_combinations): eval.append([]) for idx in range(start_idx, end_idx): cfg = sweeper.parse(idx) log_dir = cfg.get_logdir() log_path = os.path.join(log_dir, 'steps_log') with open(log_path, "r") as f: lines = f.readlines() if len(lines) == 0: continue # ugly parse based on the log_file format num_steps = int(lines[-1].split("|")[1].split(":")[1]) episodes = int(lines[-1].split("|")[2].split(":")[1]) if cfg.max_steps == num_steps: assert idx % sweeper.total_combinations == cfg.param_setting eval[idx % sweeper.total_combinations].append(episodes) summary = list( map(lambda x: (x[0], np.mean(x[1]), np.std(x[1]), len(x[1])), enumerate(eval))) summary = sorted(summary, key=lambda s: s[1], reverse=True) for idx, mean, std, num_runs in summary: print( "Param Setting # {:>3d} | Average num of episodes: {:>10.2f} +/- {:>5.2f} ({:>2d} runs) {} | " .format(idx, mean, std, num_runs, sweeper.param_setting_from_id(idx)))