def retro_analyze_trials(predir): '''Retro-analyze all trial level datas.''' logger.info('Retro-analyzing trials from file') from slm_lab.experiment.control import Trial filenames = ps.filter_(os.listdir(predir), lambda filename: filename.endswith('_trial_df.csv')) for idx, filename in enumerate(filenames): filepath = f'{predir}/{filename}' prepath = filepath.replace('_trial_df.csv', '') spec, info_space = util.prepath_to_spec_info_space(prepath) trial_index, _ = util.prepath_to_idxs(prepath) trial = Trial(spec, info_space) trial.session_data_dict = session_data_dict_from_file( predir, trial_index, ps.get(info_space, 'ckpt')) # zip only at the last zip = (idx == len(filenames) - 1) trial_fitness_df = analysis.analyze_trial(trial, zip) # write trial_data that was written from ray search trial_data_filepath = filepath.replace('_trial_df.csv', '_trial_data.json') if os.path.exists(trial_data_filepath): fitness_vec = trial_fitness_df.iloc[0].to_dict() fitness = analysis.calc_fitness(trial_fitness_df) trial_data = util.read(trial_data_filepath) trial_data.update({ **fitness_vec, 'fitness': fitness, 'trial_index': trial_index, }) util.write(trial_data, trial_data_filepath)
def retro_analyze_sessions(predir): '''Retro-analyze all session level datas.''' logger.info('Retro-analyzing sessions from file') from slm_lab.experiment.control import Session, SpaceSession for filename in os.listdir(predir): # to account for both types of session_df if filename.endswith('_session_df.csv'): body_df_kind = 'eval' # from body.eval_df prefix = '' is_session_df = True elif filename.endswith('_trainsession_df.csv'): body_df_kind = 'train' # from body.train_df prefix = 'train' is_session_df = True else: is_session_df = False if is_session_df: prepath = f'{predir}/{filename}'.replace( f'_{prefix}session_df.csv', '') spec, info_space = util.prepath_to_spec_info_space(prepath) trial_index, session_index = util.prepath_to_idxs(prepath) SessionClass = Session if spec_util.is_singleton( spec) else SpaceSession session = SessionClass(spec, info_space) session_data = session_data_from_file(predir, trial_index, session_index, ps.get(info_space, 'ckpt'), prefix) analysis._analyze_session(session, session_data, body_df_kind)
def plot_experiment_trials(experiment_spec, experiment_df, metrics_cols): meta_spec = experiment_spec['meta'] info_prepath = meta_spec['info_prepath'] trial_metrics_path_list = glob(f'{info_prepath}*_trial_metrics.pkl') # sort by trial id trial_metrics_path_list = list( sorted(trial_metrics_path_list, key=lambda k: util.prepath_to_idxs(k)[0])) # get trial indices to build legends trial_idxs = [ util.prepath_to_idxs(prepath)[0] for prepath in trial_metrics_path_list ] legend_list = get_trial_legends(experiment_df, trial_idxs, metrics_cols) title = f'multi trial graph: {experiment_spec["name"]}' graph_prepath = meta_spec['graph_prepath'] plot_multi_trial(trial_metrics_path_list, legend_list, title, graph_prepath) plot_multi_trial(trial_metrics_path_list, legend_list, title, graph_prepath, ma=True)
def retro_analyze_sessions(predir): '''Retro-analyze all session level datas.''' logger.info('Retro-analyzing sessions from file') from slm_lab.experiment.control import Session, SpaceSession for filename in os.listdir(predir): if filename.endswith('_session_df.csv'): prepath = f'{predir}/{filename}'.replace('_session_df.csv', '') spec, info_space = util.prepath_to_spec_info_space(prepath) trial_index, session_index = util.prepath_to_idxs(prepath) SessionClass = Session if spec_util.is_singleton( spec) else SpaceSession session = SessionClass(spec, info_space) session_data = session_data_from_file(predir, trial_index, session_index) analyze_session(session, session_data)
def retro_analyze_trials(predir): '''Retro-analyze all trial level datas.''' logger.info('Retro-analyzing trials from file') from slm_lab.experiment.control import Trial for filename in os.listdir(predir): if filename.endswith('_trial_data.json'): filepath = f'{predir}/{filename}' prepath = filepath.replace('_trial_data.json', '') spec, info_space = util.prepath_to_spec_info_space(prepath) trial_index, _ = util.prepath_to_idxs(prepath) trial = Trial(spec, info_space) trial.session_data_dict = session_data_dict_from_file( predir, trial_index) trial_fitness_df = analyze_trial(trial) # write trial_data that was written from ray search fitness_vec = trial_fitness_df.iloc[0].to_dict() fitness = calc_fitness(trial_fitness_df) trial_data = util.read(filepath) trial_data.update({ **fitness_vec, 'fitness': fitness, 'trial_index': trial_index, }) util.write(trial_data, filepath)