def retro_analyze_trials(predir):
'''Retro-analyze all trial level datas.'''
logger.info('Retro-analyzing trials from file')
from slm_lab.experiment.control import Trial
filenames = ps.filter_(os.listdir(predir),
lambda filename: filename.endswith('_trial_df.csv'))
for idx, filename in enumerate(filenames):
filepath = f'{predir}/{filename}'
prepath = filepath.replace('_trial_df.csv', '')
spec, info_space = util.prepath_to_spec_info_space(prepath)
trial_index, _ = util.prepath_to_idxs(prepath)
trial = Trial(spec, info_space)
trial.session_data_dict = session_data_dict_from_file(
predir, trial_index, ps.get(info_space, 'ckpt'))
# zip only at the last
zip = (idx == len(filenames) - 1)
trial_fitness_df = analysis.analyze_trial(trial, zip)
# write trial_data that was written from ray search
trial_data_filepath = filepath.replace('_trial_df.csv',
'_trial_data.json')
if os.path.exists(trial_data_filepath):
fitness_vec = trial_fitness_df.iloc[0].to_dict()
fitness = analysis.calc_fitness(trial_fitness_df)
trial_data = util.read(trial_data_filepath)
trial_data.update({
**fitness_vec,
'fitness': fitness,
'trial_index': trial_index,
})
util.write(trial_data, trial_data_filepath)
def retro_analyze_sessions(predir):
'''Retro-analyze all session level datas.'''
logger.info('Retro-analyzing sessions from file')
from slm_lab.experiment.control import Session, SpaceSession
for filename in os.listdir(predir):
# to account for both types of session_df
if filename.endswith('_session_df.csv'):
body_df_kind = 'eval' # from body.eval_df
prefix = ''
is_session_df = True
elif filename.endswith('_trainsession_df.csv'):
body_df_kind = 'train' # from body.train_df
prefix = 'train'
is_session_df = True
else:
is_session_df = False
if is_session_df:
prepath = f'{predir}/{filename}'.replace(
f'_{prefix}session_df.csv', '')
spec, info_space = util.prepath_to_spec_info_space(prepath)
trial_index, session_index = util.prepath_to_idxs(prepath)
SessionClass = Session if spec_util.is_singleton(
spec) else SpaceSession
session = SessionClass(spec, info_space)
session_data = session_data_from_file(predir, trial_index,
session_index,
ps.get(info_space,
'ckpt'), prefix)
analysis._analyze_session(session, session_data, body_df_kind)
def plot_experiment_trials(experiment_spec, experiment_df, metrics_cols):
meta_spec = experiment_spec['meta']
info_prepath = meta_spec['info_prepath']
trial_metrics_path_list = glob(f'{info_prepath}*_trial_metrics.pkl')
# sort by trial id
trial_metrics_path_list = list(
sorted(trial_metrics_path_list,
key=lambda k: util.prepath_to_idxs(k)[0]))
# get trial indices to build legends
trial_idxs = [
util.prepath_to_idxs(prepath)[0] for prepath in trial_metrics_path_list
]
legend_list = get_trial_legends(experiment_df, trial_idxs, metrics_cols)
title = f'multi trial graph: {experiment_spec["name"]}'
graph_prepath = meta_spec['graph_prepath']
plot_multi_trial(trial_metrics_path_list, legend_list, title,
graph_prepath)
plot_multi_trial(trial_metrics_path_list,
legend_list,
title,
graph_prepath,
ma=True)
def retro_analyze_sessions(predir):
'''Retro-analyze all session level datas.'''
logger.info('Retro-analyzing sessions from file')
from slm_lab.experiment.control import Session, SpaceSession
for filename in os.listdir(predir):
if filename.endswith('_session_df.csv'):
prepath = f'{predir}/{filename}'.replace('_session_df.csv', '')
spec, info_space = util.prepath_to_spec_info_space(prepath)
trial_index, session_index = util.prepath_to_idxs(prepath)
SessionClass = Session if spec_util.is_singleton(
spec) else SpaceSession
session = SessionClass(spec, info_space)
session_data = session_data_from_file(predir, trial_index,
session_index)
analyze_session(session, session_data)
def retro_analyze_trials(predir):
'''Retro-analyze all trial level datas.'''
logger.info('Retro-analyzing trials from file')
from slm_lab.experiment.control import Trial
for filename in os.listdir(predir):
if filename.endswith('_trial_data.json'):
filepath = f'{predir}/{filename}'
prepath = filepath.replace('_trial_data.json', '')
spec, info_space = util.prepath_to_spec_info_space(prepath)
trial_index, _ = util.prepath_to_idxs(prepath)
trial = Trial(spec, info_space)
trial.session_data_dict = session_data_dict_from_file(
predir, trial_index)
trial_fitness_df = analyze_trial(trial)
# write trial_data that was written from ray search
fitness_vec = trial_fitness_df.iloc[0].to_dict()
fitness = calc_fitness(trial_fitness_df)
trial_data = util.read(filepath)
trial_data.update({
**fitness_vec,
'fitness': fitness,
'trial_index': trial_index,
})
util.write(trial_data, filepath)