def retro_analyze_trials(predir):
'''Retro-analyze all trial level datas.'''
logger.info('Retro-analyzing trials from file')
from slm_lab.experiment.control import Trial
for filename in os.listdir(predir):
if filename.endswith('_trial_data.json'):
filepath = f'{predir}/{filename}'
tn = filename.replace('_trial_data.json', '').split('_')[-1]
trial_index = int(tn[1:])
# mock trial
spec, info_space = mock_info_space_spec(predir, trial_index)
trial = Trial(spec, info_space)
session_data_dict = session_data_dict_from_file(
predir, trial_index)
trial.session_data_dict = session_data_dict
trial_fitness_df = analyze_trial(trial)
# write trial_data that was written from ray search
fitness_vec = trial_fitness_df.iloc[0].to_dict()
fitness = calc_fitness(trial_fitness_df)
trial_data = util.read(filepath)
trial_data.update({
**fitness_vec,
'fitness': fitness,
'trial_index': trial_index,
})
util.write(trial_data, filepath)
def retro_analyze_trials(predir):
'''Retro-analyze all trial level datas.'''
logger.info('Retro-analyzing trials from file')
from slm_lab.experiment.control import Trial
filenames = ps.filter_(os.listdir(predir),
lambda filename: filename.endswith('_trial_df.csv'))
for idx, filename in enumerate(filenames):
filepath = f'{predir}/{filename}'
prepath = filepath.replace('_trial_df.csv', '')
spec, info_space = util.prepath_to_spec_info_space(prepath)
trial_index, _ = util.prepath_to_idxs(prepath)
trial = Trial(spec, info_space)
trial.session_data_dict = session_data_dict_from_file(
predir, trial_index, ps.get(info_space, 'ckpt'))
# zip only at the last
zip = (idx == len(filenames) - 1)
trial_fitness_df = analysis.analyze_trial(trial, zip)
# write trial_data that was written from ray search
trial_data_filepath = filepath.replace('_trial_df.csv',
'_trial_data.json')
if os.path.exists(trial_data_filepath):
fitness_vec = trial_fitness_df.iloc[0].to_dict()
fitness = analysis.calc_fitness(trial_fitness_df)
trial_data = util.read(trial_data_filepath)
trial_data.update({
**fitness_vec,
'fitness': fitness,
'trial_index': trial_index,
})
util.write(trial_data, trial_data_filepath)
def save_experiment_data(spec, info_space, experiment_df, experiment_fig):
'''Save the experiment data: best_spec, experiment_df, experiment_graph.'''
prepath = util.get_prepath(spec, info_space, unit='experiment')
logger.info(f'Saving experiment data to {prepath}')
util.write(experiment_df, f'{prepath}_experiment_df.csv')
viz.save_image(experiment_fig, f'{prepath}_experiment_graph.png')
# TODO tmp hack
plot_best_sessions(experiment_df, prepath)
def save_experiment_data(info_space, best_spec, experiment_df, experiment_fig):
'''Save the experiment data: best_spec, experiment_df, experiment_graph.'''
spec_name = best_spec['name']
predir = f'data/{spec_name}_{info_space.experiment_ts}'
prename = f'{spec_name}'
logger.info(f'Saving experiment data to {predir}')
util.write(best_spec, f'{predir}/{prename}_best_spec.json')
util.write(experiment_df, f'{predir}/{prename}_experiment_df.csv')
viz.save_image(experiment_fig, f'{predir}/{prename}_experiment_graph.png')
plot_best_sessions(experiment_df, predir, prename)
def save_trial_data(info_space, trial_spec, trial_fitness_df):
'''Save the trial data: spec, trial_fitness_df.'''
spec_name = trial_spec['name']
trial_index = info_space.coor['trial']
predir = f'data/{spec_name}_{info_space.experiment_ts}'
prename = f'{spec_name}_t{trial_index}'
logger.info(f'Saving trial {trial_index} data to {predir}')
util.write(trial_spec, f'{predir}/{prename}_spec.json')
# TODO trial data is composed of saved session data files
util.write(trial_fitness_df, f'{predir}/{prename}_trial_fitness_df.csv')
def save_trial_data(spec, info_space, trial_fitness_df, trial_fig):
'''Save the trial data: spec, trial_fitness_df.'''
prepath = util.get_prepath(spec, info_space, unit='trial')
logger.info(f'Saving trial data to {prepath}')
util.write(trial_fitness_df, f'{prepath}_trial_fitness_df.csv')
viz.save_image(trial_fig, f'{prepath}_trial_graph.png')
if util.get_lab_mode() == ('train', 'eval'):
predir, _, _, _, _, _ = util.prepath_split(prepath)
shutil.make_archive(predir, 'zip', predir)
logger.info(f'All trial data zipped to {predir}.zip')
def save_experiment_data(spec, info_space, experiment_df, experiment_fig):
'''Save the experiment data: best_spec, experiment_df, experiment_graph.'''
prepath = util.get_prepath(spec, info_space, unit='experiment')
logger.info(f'Saving experiment data to {prepath}')
util.write(experiment_df, f'{prepath}_experiment_df.csv')
viz.save_image(experiment_fig, f'{prepath}_experiment_graph.png')
# zip for ease of upload
predir, _, _, _, _, _ = util.prepath_split(prepath)
shutil.make_archive(predir, 'zip', predir)
logger.info(f'All experiment data zipped to {predir}.zip')
def run_trial(experiment, config):
trial_index = config.pop('trial_index')
spec = spec_from_config(experiment, config)
info_space = deepcopy(experiment.info_space)
info_space.set('trial', trial_index)
trial_fitness_df = experiment.init_trial_and_run(spec, info_space)
fitness_vec = trial_fitness_df.iloc[0].to_dict()
fitness = analysis.calc_fitness(trial_fitness_df)
trial_data = {**config, **fitness_vec, 'fitness': fitness, 'trial_index': trial_index}
prepath = util.get_prepath(spec, info_space, unit='trial')
util.write(trial_data, f'{prepath}_trial_data.json')
return trial_data
def analyze_experiment(spec, trial_data_dict):
'''Analyze experiment and save data'''
info_prepath = spec['meta']['info_prepath']
util.write(trial_data_dict, f'{info_prepath}_trial_data_dict.json')
# calculate experiment df
experiment_df = calc_experiment_df(trial_data_dict, info_prepath)
# plot graph
viz.plot_experiment(spec, experiment_df, METRICS_COLS)
# zip files
predir, _, _, _, _, _ = util.prepath_split(info_prepath)
shutil.make_archive(predir, 'zip', predir)
logger.info(f'All experiment data zipped to {predir}.zip')
return experiment_df
def analyze_session(session_spec, session_df, df_mode, plot=True):
'''Analyze session and save data, then return metrics. Note there are 2 types of session_df: body.eval_df and body.train_df'''
info_prepath = session_spec['meta']['info_prepath']
session_df = session_df.copy()
assert len(session_df) > 2, f'Need more than 2 datapoint to calculate metrics' # first datapoint at frame 0 is empty
util.write(session_df, f'{info_prepath}_session_df_{df_mode}.csv')
# calculate metrics
session_metrics = calc_session_metrics(session_df, ps.get(session_spec, 'env.0.name'), info_prepath, df_mode)
if plot:
# plot graph
viz.plot_session(session_spec, session_metrics, session_df, df_mode)
viz.plot_session(session_spec, session_metrics, session_df, df_mode, ma=True)
return session_metrics
def save_session_data(session_spec, session_df, session_fig):
'''
Save the session data: df, plot.
session_df is multi-indexed with (a,e,b), 3 extra levels
to read, use:
session_df = util.read(filepath, header=[0, 1, 2, 3])
session_data = util.session_df_to_data(session_df)
'''
# TODO generalize to use experiment timestamp, id, sesison coor in info space, to replace timestamp
spec_name = session_spec['name']
prepath = f'data/{spec_name}/{spec_name}_{util.get_timestamp()}'
logger.info(f'Saving session data to {prepath}_*')
util.write(session_df, f'{prepath}_session_df.csv')
viz.save_image(session_fig, f'{prepath}_session_graph.png')
def calc_experiment_df(trial_data_dict, info_prepath=None):
'''Collect all trial data (metrics and config) from trials into a dataframe'''
experiment_df = pd.DataFrame(trial_data_dict).transpose()
cols = METRICS_COLS
config_cols = sorted(ps.difference(experiment_df.columns.tolist(), cols))
sorted_cols = config_cols + cols
experiment_df = experiment_df.reindex(sorted_cols, axis=1)
experiment_df.sort_values(by=['strength'], ascending=False, inplace=True)
if info_prepath is not None:
util.write(experiment_df, f'{info_prepath}_experiment_df.csv')
# save important metrics in info_prepath directly
util.write(experiment_df,
f'{info_prepath.replace("info/", "")}_experiment_df.csv')
return experiment_df
def run_trial(experiment, config):
trial_index = config.pop('trial_index')
spec = spec_from_config(experiment, config)
info_space = deepcopy(experiment.info_space)
info_space.set('trial', trial_index)
trial_fitness_df = experiment.init_trial_and_run(spec, info_space)
fitness_vec = trial_fitness_df.iloc[0].to_dict()
fitness = analysis.calc_fitness(trial_fitness_df)
trial_data = {
**config, **fitness_vec, 'fitness': fitness, 'trial_index': trial_index,
}
prepath = analysis.get_prepath(spec, info_space, unit='trial')
util.write(trial_data, f'{prepath}_trial_data.json')
return trial_data
def save_session_data(spec, info_space, session_data, session_fitness_df, session_fig, body_df_kind='eval'):
'''
Save the session data: session_df, session_fitness_df, session_graph.
session_data is saved as session_df; multi-indexed with (a,e,b), 3 extra levels
to read, use:
session_df = util.read(filepath, header=[0, 1, 2, 3], index_col=0)
session_data = util.session_df_to_data(session_df)
'''
prepath = util.get_prepath(spec, info_space, unit='session')
logger.info(f'Saving session data to {prepath}')
prefix = 'train' if body_df_kind == 'train' else ''
if 'retro_analyze' not in os.environ['PREPATH']:
save_session_df(session_data, f'{prepath}_{prefix}session_df.csv', info_space)
util.write(session_fitness_df, f'{prepath}_{prefix}session_fitness_df.csv')
viz.save_image(session_fig, f'{prepath}_{prefix}session_graph.png')
def analyze_session(session_spec, session_df, df_mode, plot=True):
'''Analyze session and save data, then return metrics. Note there are 2 types of session_df: body.eval_df and body.train_df'''
info_prepath = session_spec['meta']['info_prepath']
session_df = session_df.copy() # prevent modification
assert len(session_df) > 2, f'Need more than 2 datapoint to calculate metrics' # first datapoint at frame 0 is empty
util.write(session_df, util.get_session_df_path(session_spec, df_mode))
# calculate metrics
session_metrics = calc_session_metrics(session_df, ps.get(session_spec, 'env.0.name'), info_prepath, df_mode)
if plot:
# plot graph
viz.plot_session(session_spec, session_metrics, session_df, df_mode)
viz.plot_session(session_spec, session_metrics, session_df, df_mode, ma=True)
# manually shut down orca server to avoid zombie processes
viz.pio.orca.shutdown_server()
return session_metrics
def save_session_data(spec, info_space, session_data, session_fitness_df, session_fig):
'''
Save the session data: session_df, session_fitness_df, session_graph.
session_data is saved as session_df; multi-indexed with (a,e,b), 3 extra levels
to read, use:
session_df = util.read(filepath, header=[0, 1, 2, 3])
session_data = util.session_df_to_data(session_df)
'''
prepath = util.get_prepath(spec, info_space, unit='session')
logger.info(f'Saving session data to {prepath}')
if 'retro_analyze' not in os.environ['PREPATH']:
session_df = pd.concat(session_data, axis=1)
util.write(session_df, f'{prepath}_session_df.csv')
util.write(session_fitness_df, f'{prepath}_session_fitness_df.csv')
viz.save_image(session_fig, f'{prepath}_session_graph.png')
def analyze_experiment(spec, trial_data_dict):
'''Analyze experiment and save data'''
info_prepath = spec['meta']['info_prepath']
util.write(trial_data_dict, f'{info_prepath}_trial_data_dict.json')
# calculate experiment df
experiment_df = calc_experiment_df(trial_data_dict, info_prepath)
# plot graph
viz.plot_experiment(spec, experiment_df, METRICS_COLS)
viz.plot_experiment_trials(spec, experiment_df, METRICS_COLS)
# manually shut down orca server to avoid zombie processes
viz.pio.orca.shutdown_server()
# zip files
predir, _, _, _, _, _ = util.prepath_split(info_prepath)
zipdir = util.smart_path(predir)
shutil.make_archive(zipdir, 'zip', zipdir)
logger.info(f'All experiment data zipped to {predir}.zip')
return experiment_df
def save_session_df(session_data, prepath, info_space):
'''Save session_df, and if is in eval mode, modify it and save with append'''
filepath = f'{prepath}_session_df.csv'
if util.get_lab_mode() in ('enjoy', 'eval'):
ckpt = util.find_ckpt(info_space.eval_model_prepath)
epi = int(re.search('epi(\d+)', ckpt)[1])
totalt = int(re.search('totalt(\d+)', ckpt)[1])
session_df = pd.concat(session_data, axis=1)
eval_session_df = pd.DataFrame(data=[session_df.mean()])
for aeb in util.get_df_aeb_list(eval_session_df):
eval_session_df.loc[:, aeb + ('epi', )] = epi
eval_session_df.loc[:, aeb + ('total_t', )] = totalt
# if eval, save with append mode
header = not os.path.exists(filepath)
with open(filepath, 'a') as f:
eval_session_df.to_csv(f, header=header)
else:
session_df = pd.concat(session_data, axis=1)
util.write(session_df, filepath)
def save_session_data(spec, info_space, session_mdp_data, session_data, session_fitness_df, session_fig):
'''
Save the session data: session_mdp_df, session_df, session_fitness_df, session_graph.
session_data is saved as session_df; multi-indexed with (a,e,b), 3 extra levels
to read, use:
session_df = util.read(filepath, header=[0, 1, 2, 3])
session_data = util.session_df_to_data(session_df)
Likewise for session_mdp_df
'''
prepath = util.get_prepath(spec, info_space, unit='session')
logger.info(f'Saving session data to {prepath}')
if session_mdp_data is not None: # not from retro analysis
session_mdp_df = pd.concat(session_mdp_data, axis=1)
session_df = pd.concat(session_data, axis=1)
# TODO reactivate saving when get to the transition matrix research
# util.write(session_mdp_df, f'{prepath}_session_mdp_df.csv')
util.write(session_df, f'{prepath}_session_df.csv')
util.write(session_fitness_df, f'{prepath}_session_fitness_df.csv')
viz.save_image(session_fig, f'{prepath}_session_graph.png')
def get_random_baseline(env_name):
'''Get a single random baseline for env; if does not exist in file, generate live and update the file'''
random_baseline = util.read(FILEPATH)
if env_name in random_baseline:
baseline = random_baseline[env_name]
else:
try:
logger.info(f'Generating random baseline for {env_name}')
baseline = gen_random_baseline(env_name, NUM_EVAL)
except Exception as e:
logger.warning(
f'Cannot start env: {env_name}, skipping random baseline generation'
)
baseline = None
# update immediately
logger.info(f'Updating new random baseline in {FILEPATH}')
random_baseline[env_name] = baseline
util.write(random_baseline, FILEPATH)
return baseline
def generate_specs(spec, const='agent'):
'''
Generate benchmark specs with compatible discrete/continuous/both types:
- take a spec
- for each in benchmark envs
- use the template env spec to update spec
- append to benchmark specs
Interchange agent and env for the reversed benchmark.
'''
if const == 'agent':
const_name = _.get(spec, 'agent.0.algorithm.name')
variant = 'env'
else:
const_name = _.get(spec, 'env.0.name')
variant = 'agent'
filepath = f'{spec_util.SPEC_DIR}/benchmark_{const_name}.json'
if os.path.exists(filepath):
logger.info(
f'Benchmark for {const_name} exists at {filepath} already, not overwriting.'
)
benchmark_specs = util.read(filepath)
return benchmark_specs
logger.info(f'Generating benchmark for {const_name}')
benchmark_variants = []
benchmark_specs = {}
for dist_cont, const_names in BENCHMARK[const].items():
if const_name in const_names:
benchmark_variants.extend(BENCHMARK[variant][dist_cont])
for vary_name in benchmark_variants:
vary_spec = ENV_TEMPLATES[vary_name]
spec_name = f'{const_name}_{vary_name}'
benchmark_spec = spec.copy()
benchmark_spec['name'] = spec_name
benchmark_spec[variant] = [vary_spec]
benchmark_specs[spec_name] = benchmark_spec
util.write(benchmark_specs, filepath)
logger.info(f'Benchmark for {const_name} written to {filepath}.')
return benchmark_specs
def save_session_data(spec, info_space, session_mdp_data, session_data, session_fitness_df, session_fig):
'''
Save the session data: session_mdp_df, session_df, session_fitness_df, session_graph.
session_data is saved as session_df; multi-indexed with (a,e,b), 3 extra levels
to read, use:
session_df = util.read(filepath, header=[0, 1, 2, 3])
session_data = util.session_df_to_data(session_df)
Likewise for session_mdp_df
'''
prepath = util.get_prepath(spec, info_space, unit='session')
logger.info(f'Saving session data to {prepath}')
if session_mdp_data is not None: # not from retro analysis
session_mdp_df = pd.concat(session_mdp_data, axis=1)
session_df = pd.concat(session_data, axis=1)
# TODO reactivate saving when get to the transition matrix research
# util.write(session_mdp_df, f'{prepath}_session_mdp_df.csv')
util.write(session_df, f'{prepath}_session_df.csv')
util.write(session_fitness_df, f'{prepath}_session_fitness_df.csv')
# TODO replaced by plot_best_sessions until Feb 2018 https://github.com/plotly/plotly.py/issues/880
if os.environ.get('lab_mode') == 'train':
viz.save_image(session_fig, f'{prepath}_session_graph.png')
def save_session_df(session_data, filepath, info_space):
'''Save session_df, and if is in eval mode, modify it and save with append'''
if util.in_eval_lab_modes():
ckpt = util.find_ckpt(info_space.eval_model_prepath)
epi = int(re.search('epi(\d+)', ckpt)[1])
totalt = int(re.search('totalt(\d+)', ckpt)[1])
session_df = pd.concat(session_data, axis=1)
mean_sr = session_df.mean()
mean_sr.name = totalt # set index to prevent all being the same
eval_session_df = pd.DataFrame(data=[mean_sr])
# set sr name too, to total_t
for aeb in util.get_df_aeb_list(eval_session_df):
eval_session_df.loc[:, aeb + ('epi',)] = epi
eval_session_df.loc[:, aeb + ('total_t',)] = totalt
# if eval, save with append mode
header = not os.path.exists(filepath)
with open(filepath, 'a') as f:
eval_session_df.to_csv(f, header=header)
else:
session_df = pd.concat(session_data, axis=1)
util.write(session_df, filepath)
def generate_specs(spec, const='agent'):
'''
Generate benchmark specs with compatible discrete/continuous/both types:
- take a spec
- for each in benchmark envs
- use the template env spec to update spec
- append to benchmark specs
Interchange agent and env for the reversed benchmark.
'''
if const == 'agent':
const_name = ps.get(spec, 'agent.0.algorithm.name')
variant = 'env'
else:
const_name = ps.get(spec, 'env.0.name')
variant = 'agent'
filepath = f'{spec_util.SPEC_DIR}/benchmark_{const_name}.json'
if os.path.exists(filepath):
logger.info(f'Benchmark for {const_name} exists at {filepath} already, not overwriting.')
benchmark_specs = util.read(filepath)
return benchmark_specs
logger.info(f'Generating benchmark for {const_name}')
benchmark_variants = []
benchmark_specs = {}
for dist_cont, const_names in BENCHMARK[const].items():
if const_name in const_names:
benchmark_variants.extend(BENCHMARK[variant][dist_cont])
for vary_name in benchmark_variants:
vary_spec = ENV_TEMPLATES[vary_name]
spec_name = f'{const_name}_{vary_name}'
benchmark_spec = spec.copy()
benchmark_spec['name'] = spec_name
benchmark_spec[variant] = [vary_spec]
benchmark_specs[spec_name] = benchmark_spec
util.write(benchmark_specs, filepath)
logger.info(f'Benchmark for {const_name} written to {filepath}.')
return benchmark_specs
def calc_session_metrics(session_df,
env_name,
info_prepath=None,
df_mode=None):
'''
Calculate the session metrics: strength, efficiency, stability
@param DataFrame:session_df Dataframe containing reward, frame, opt_step
@param str:env_name Name of the environment to get its random baseline
@param str:info_prepath Optional info_prepath to auto-save the output to
@param str:df_mode Optional df_mode to save with info_prepath
@returns dict:metrics Consists of scalar metrics and series local metrics
'''
rand_bl = random_baseline.get_random_baseline(env_name)
mean_rand_returns = rand_bl['mean']
mean_returns = session_df['total_reward']
frames = session_df['frame']
opt_steps = session_df['opt_step']
final_return_ma = mean_returns[-viz.PLOT_MA_WINDOW:].mean()
str_, local_strs = calc_strength(mean_returns, mean_rand_returns)
max_str, final_str = local_strs.max(), local_strs.iloc[-1]
sample_eff, local_sample_effs = calc_efficiency(local_strs, frames)
train_eff, local_train_effs = calc_efficiency(local_strs, opt_steps)
sta, local_stas = calc_stability(local_strs)
# all the scalar session metrics
scalar = {
'final_return_ma': final_return_ma,
'strength': str_,
'max_strength': max_str,
'final_strength': final_str,
'sample_efficiency': sample_eff,
'training_efficiency': train_eff,
'stability': sta,
}
# all the session local metrics
local = {
'mean_returns': mean_returns,
'strengths': local_strs,
'sample_efficiencies': local_sample_effs,
'training_efficiencies': local_train_effs,
'stabilities': local_stas,
'frames': frames,
'opt_steps': opt_steps,
}
metrics = {
'scalar': scalar,
'local': local,
}
if info_prepath is not None: # auto-save if info_prepath is given
util.write(metrics, f'{info_prepath}_session_metrics_{df_mode}.pkl')
util.write(scalar,
f'{info_prepath}_session_metrics_scalar_{df_mode}.json')
# save important metrics in info_prepath directly
util.write(
scalar,
f'{info_prepath.replace("info/", "")}_session_metrics_scalar_{df_mode}.json'
)
return metrics
def retro_analyze_trials(predir):
'''Retro-analyze all trial level datas.'''
logger.info('Retro-analyzing trials from file')
from slm_lab.experiment.control import Trial
for filename in os.listdir(predir):
if filename.endswith('_trial_data.json'):
filepath = f'{predir}/{filename}'
tn = filename.replace('_trial_data.json', '').split('_')[-1]
trial_index = int(tn[1:])
# mock trial
spec, info_space = mock_info_space_spec(predir, trial_index)
trial = Trial(spec, info_space)
session_data_dict = session_data_dict_from_file(predir, trial_index)
trial.session_data_dict = session_data_dict
trial_fitness_df = analyze_trial(trial)
# write trial_data that was written from ray search
fitness_vec = trial_fitness_df.iloc[0].to_dict()
fitness = calc_fitness(trial_fitness_df)
trial_data = util.read(filepath)
trial_data.update({
**fitness_vec, 'fitness': fitness, 'trial_index': trial_index,
})
util.write(trial_data, filepath)
def calc_trial_metrics(session_metrics_list, info_prepath=None):
'''
Calculate the trial metrics: mean(strength), mean(efficiency), mean(stability), consistency
@param list:session_metrics_list The metrics collected from each session; format: {session_index: {'scalar': {...}, 'local': {...}}}
@param str:info_prepath Optional info_prepath to auto-save the output to
@returns dict:metrics Consists of scalar metrics and series local metrics
'''
# calculate mean of session metrics
scalar_list = [sm['scalar'] for sm in session_metrics_list]
mean_scalar = pd.DataFrame(scalar_list).mean().to_dict()
mean_returns_list = [sm['local']['mean_returns'] for sm in session_metrics_list]
local_strs_list = [sm['local']['strengths'] for sm in session_metrics_list]
local_se_list = [sm['local']['sample_efficiencies'] for sm in session_metrics_list]
local_te_list = [sm['local']['training_efficiencies'] for sm in session_metrics_list]
local_sta_list = [sm['local']['stabilities'] for sm in session_metrics_list]
frames = session_metrics_list[0]['local']['frames']
opt_steps = session_metrics_list[0]['local']['opt_steps']
# calculate consistency
con, local_cons = calc_consistency(local_strs_list)
# all the scalar trial metrics
scalar = {
'final_return_ma': mean_scalar['final_return_ma'],
'strength': mean_scalar['strength'],
'max_strength': mean_scalar['max_strength'],
'final_strength': mean_scalar['final_strength'],
'sample_efficiency': mean_scalar['sample_efficiency'],
'training_efficiency': mean_scalar['training_efficiency'],
'stability': mean_scalar['stability'],
'consistency': con,
}
assert set(scalar.keys()) == set(METRICS_COLS)
# for plotting: gather all local series of sessions
local = {
'mean_returns': mean_returns_list,
'strengths': local_strs_list,
'sample_efficiencies': local_se_list,
'training_efficiencies': local_te_list,
'stabilities': local_sta_list,
'consistencies': local_cons, # this is a list
'frames': frames,
'opt_steps': opt_steps,
}
metrics = {
'scalar': scalar,
'local': local,
}
if info_prepath is not None: # auto-save if info_prepath is given
util.write(metrics, f'{info_prepath}_trial_metrics.pkl')
util.write(scalar, f'{info_prepath}_trial_metrics_scalar.json')
# save important metrics in info_prepath directly
util.write(scalar, f'{info_prepath.replace("info/", "")}_trial_metrics_scalar.json')
return metrics
def save_session_data(info_space, session_spec, session_mdp_data, session_data,
session_fitness_df, session_fig):
'''
Save the session data: session_mdp_df, session_df, session_fitness_df, session_graph.
session_data is saved as session_df; multi-indexed with (a,e,b), 3 extra levels
to read, use:
session_df = util.read(filepath, header=[0, 1, 2, 3])
session_data = util.session_df_to_data(session_df)
Likewise for session_mdp_df
'''
spec_name = session_spec['name']
trial_index = info_space.coor['trial']
session_index = info_space.coor['session']
predir = f'data/{spec_name}_{info_space.experiment_ts}'
prename = f'{spec_name}_t{trial_index}_s{session_index}'
logger.info(
f'Saving trial {trial_index} session {session_index} data to {predir}')
session_mdp_df = pd.concat(session_mdp_data, axis=1)
session_df = pd.concat(session_data, axis=1)
util.write(session_mdp_df, f'{predir}/{prename}_session_mdp_df.csv')
util.write(session_df, f'{predir}/{prename}_session_df.csv')
util.write(session_fitness_df,
f'{predir}/{prename}_session_fitness_df.csv')
def save_trial_data(spec, info_space, trial_fitness_df, trial_fig):
'''Save the trial data: spec, trial_fitness_df.'''
prepath = util.get_prepath(spec, info_space, unit='trial')
logger.info(f'Saving trial data to {prepath}')
util.write(trial_fitness_df, f'{prepath}_trial_fitness_df.csv')
viz.save_image(trial_fig, f'{prepath}_trial_graph.png')
def test_write_read_as_plain_list(test_str, filename, dtype):
data_path = f'test/fixture/common/util/{filename}'
util.write(test_str, util.smart_path(data_path))
assert os.path.exists(data_path)
data_dict = util.read(util.smart_path(data_path))
assert isinstance(data_dict, dtype)
def test_write_read_as_df(test_df, filename, dtype):
data_path = f'test/fixture/common/util/{filename}'
util.write(test_df, util.smart_path(data_path))
assert os.path.exists(data_path)
data_df = util.read(util.smart_path(data_path))
assert isinstance(data_df, dtype)
def save(spec, unit='experiment'):
'''Save spec to proper path. Called at Experiment or Trial init.'''
prepath = util.get_prepath(spec, unit)
util.write(spec, f'{prepath}_spec.json')
def test_write_read_as_plain_list(test_str, filename, dtype):
data_path = f'test/fixture/lib/util/{filename}'
util.write(test_str, util.smart_path(data_path))
assert os.path.exists(data_path)
data_dict = util.read(util.smart_path(data_path))
assert isinstance(data_dict, dtype)
def test_write_read_as_df(test_df, filename, dtype):
data_path = f'test/fixture/lib/util/{filename}'
util.write(test_df, util.smart_path(data_path))
assert os.path.exists(data_path)
data_df = util.read(util.smart_path(data_path))
assert isinstance(data_df, dtype)
def save_spec(spec, info_space, unit='experiment'):
'''Save spec to proper path. Called at Experiment or Trial init.'''
prepath = util.get_prepath(spec, info_space, unit)
util.write(spec, f'{prepath}_spec.json')
def save_trial_data(spec, info_space, trial_fitness_df):
'''Save the trial data: spec, trial_fitness_df.'''
prepath = get_prepath(spec, info_space, unit='trial')
logger.info(f'Saving trial data to {prepath}')
util.write(trial_fitness_df, f'{prepath}_trial_fitness_df.csv')
def save_experiment_data(spec, info_space, experiment_df, experiment_fig):
'''Save the experiment data: best_spec, experiment_df, experiment_graph.'''
prepath = util.get_prepath(spec, info_space, unit='experiment')
logger.info(f'Saving experiment data to {prepath}')
util.write(experiment_df, f'{prepath}_experiment_df.csv')
viz.save_image(experiment_fig, f'{prepath}_experiment_graph.png')
def save_trial_data(spec, info_space, trial_fitness_df, trial_fig):
'''Save the trial data: spec, trial_fitness_df.'''
prepath = util.get_prepath(spec, info_space, unit='trial')
logger.info(f'Saving trial data to {prepath}')
util.write(trial_fitness_df, f'{prepath}_trial_fitness_df.csv')
viz.save_image(trial_fig, f'{prepath}_trial_graph.png')
