def run(res_dir, res_id):
"""
Function to aggregate the benchmark results from running in SLURM mode.
Args:
res_dir (str): path to the result directory;
res_id (str): log id of the result directory.
"""
work_dir = Path(res_dir, res_id)
# check if results are aggregated
dir_name = 'run'
run_dirs = list(work_dir.glob('{}_*'.format(dir_name)))
print(run_dirs)
has_entropy = (work_dir / '{}_0/entropy.pkl'.format(dir_name)).exists()
print('has entropy:', has_entropy)
J = list()
R = list()
V = list()
E = list()
best_J = float("-inf")
best_stats = None
best_agent = None
skip_cnt = 0
for run_dir in run_dirs:
logger = BenchmarkLogger(log_dir=str(run_dir.parent), log_id=str(run_dir.name), use_timestamp=False)
print(run_dir, end=' ')
if not logger.exists_best_agent():
print("ABORT")
skip_cnt += 1
continue
else:
print("EXISTS")
J.extend(logger.load_J())
R.extend(logger.load_R())
V.extend(logger.load_V())
if has_entropy:
E.extend(logger.load_entropy())
stats = logger.load_stats()
if stats['best_J'] > best_J:
best_stats = stats
best_agent = logger.load_best_agent()
if skip_cnt > 0: print('NUMBER OF FAILED RUNS:', '{}/{}'.format(skip_cnt, len(run_dirs)))
print('Name:', res_id)
print('Directory:', res_dir)
logger = BenchmarkLogger(log_dir=res_dir, log_id=res_id, use_timestamp=False)
logger.save_J(J)
logger.save_R(R)
logger.save_V(V)
if has_entropy:
logger.save_entropy(E)
logger.save_stats(best_stats)
logger.save_best_agent(best_agent)
visualizer = BenchmarkVisualizer(logger)
visualizer.save_report()
os.path.join(log_dir, 'agent_builder.pkl'))
env_builder = BenchmarkLogger._load_pickle(
os.path.join(log_dir, 'environment_builder.pkl'))
logger = BenchmarkLogger(log_dir=log_dir,
log_id=log_id,
use_timestamp=False)
logger.info('Starting experiment.')
result = exec_run(agent_builder, env_builder, **run_args)
logger.info('Saving result.')
cmp_E = agent_builder.compute_policy_entropy
logger.save_J([result['J']])
logger.save_R([result['R']])
logger.save_V([result['V']])
if cmp_E:
logger.save_entropy([result['E']])
new_score = result['score']
new_agent = result['builders']
stats = dict(best_J=new_score[0], best_R=new_score[1], best_Q=new_score[2])
if cmp_E:
stats.update(dict(best_E=new_score[3]))
logger.save_stats(stats=stats)
logger.save_best_agent(new_agent)
logger.info('Finished execution.')