def retro_analyze_trials(predir): '''Retro-analyze all trial level datas.''' logger.info('Retro-analyzing trials from file') from slm_lab.experiment.control import Trial for filename in os.listdir(predir): if filename.endswith('_trial_data.json'): filepath = f'{predir}/{filename}' tn = filename.replace('_trial_data.json', '').split('_')[-1] trial_index = int(tn[1:]) # mock trial spec, info_space = mock_info_space_spec(predir, trial_index) trial = Trial(spec, info_space) session_data_dict = session_data_dict_from_file( predir, trial_index) trial.session_data_dict = session_data_dict trial_fitness_df = analyze_trial(trial) # write trial_data that was written from ray search fitness_vec = trial_fitness_df.iloc[0].to_dict() fitness = calc_fitness(trial_fitness_df) trial_data = util.read(filepath) trial_data.update({ **fitness_vec, 'fitness': fitness, 'trial_index': trial_index, }) util.write(trial_data, filepath)
def retro_analyze_trials(predir): '''Retro-analyze all trial level datas.''' logger.info('Retro-analyzing trials from file') from slm_lab.experiment.control import Trial filenames = ps.filter_(os.listdir(predir), lambda filename: filename.endswith('_trial_df.csv')) for idx, filename in enumerate(filenames): filepath = f'{predir}/{filename}' prepath = filepath.replace('_trial_df.csv', '') spec, info_space = util.prepath_to_spec_info_space(prepath) trial_index, _ = util.prepath_to_idxs(prepath) trial = Trial(spec, info_space) trial.session_data_dict = session_data_dict_from_file( predir, trial_index, ps.get(info_space, 'ckpt')) # zip only at the last zip = (idx == len(filenames) - 1) trial_fitness_df = analysis.analyze_trial(trial, zip) # write trial_data that was written from ray search trial_data_filepath = filepath.replace('_trial_df.csv', '_trial_data.json') if os.path.exists(trial_data_filepath): fitness_vec = trial_fitness_df.iloc[0].to_dict() fitness = analysis.calc_fitness(trial_fitness_df) trial_data = util.read(trial_data_filepath) trial_data.update({ **fitness_vec, 'fitness': fitness, 'trial_index': trial_index, }) util.write(trial_data, trial_data_filepath)
def save_experiment_data(spec, info_space, experiment_df, experiment_fig): '''Save the experiment data: best_spec, experiment_df, experiment_graph.''' prepath = util.get_prepath(spec, info_space, unit='experiment') logger.info(f'Saving experiment data to {prepath}') util.write(experiment_df, f'{prepath}_experiment_df.csv') viz.save_image(experiment_fig, f'{prepath}_experiment_graph.png') # TODO tmp hack plot_best_sessions(experiment_df, prepath)
def save_experiment_data(info_space, best_spec, experiment_df, experiment_fig): '''Save the experiment data: best_spec, experiment_df, experiment_graph.''' spec_name = best_spec['name'] predir = f'data/{spec_name}_{info_space.experiment_ts}' prename = f'{spec_name}' logger.info(f'Saving experiment data to {predir}') util.write(best_spec, f'{predir}/{prename}_best_spec.json') util.write(experiment_df, f'{predir}/{prename}_experiment_df.csv') viz.save_image(experiment_fig, f'{predir}/{prename}_experiment_graph.png') plot_best_sessions(experiment_df, predir, prename)
def save_trial_data(info_space, trial_spec, trial_fitness_df): '''Save the trial data: spec, trial_fitness_df.''' spec_name = trial_spec['name'] trial_index = info_space.coor['trial'] predir = f'data/{spec_name}_{info_space.experiment_ts}' prename = f'{spec_name}_t{trial_index}' logger.info(f'Saving trial {trial_index} data to {predir}') util.write(trial_spec, f'{predir}/{prename}_spec.json') # TODO trial data is composed of saved session data files util.write(trial_fitness_df, f'{predir}/{prename}_trial_fitness_df.csv')
def save_trial_data(spec, info_space, trial_fitness_df, trial_fig): '''Save the trial data: spec, trial_fitness_df.''' prepath = util.get_prepath(spec, info_space, unit='trial') logger.info(f'Saving trial data to {prepath}') util.write(trial_fitness_df, f'{prepath}_trial_fitness_df.csv') viz.save_image(trial_fig, f'{prepath}_trial_graph.png') if util.get_lab_mode() == ('train', 'eval'): predir, _, _, _, _, _ = util.prepath_split(prepath) shutil.make_archive(predir, 'zip', predir) logger.info(f'All trial data zipped to {predir}.zip')
def save_experiment_data(spec, info_space, experiment_df, experiment_fig): '''Save the experiment data: best_spec, experiment_df, experiment_graph.''' prepath = util.get_prepath(spec, info_space, unit='experiment') logger.info(f'Saving experiment data to {prepath}') util.write(experiment_df, f'{prepath}_experiment_df.csv') viz.save_image(experiment_fig, f'{prepath}_experiment_graph.png') # zip for ease of upload predir, _, _, _, _, _ = util.prepath_split(prepath) shutil.make_archive(predir, 'zip', predir) logger.info(f'All experiment data zipped to {predir}.zip')
def run_trial(experiment, config): trial_index = config.pop('trial_index') spec = spec_from_config(experiment, config) info_space = deepcopy(experiment.info_space) info_space.set('trial', trial_index) trial_fitness_df = experiment.init_trial_and_run(spec, info_space) fitness_vec = trial_fitness_df.iloc[0].to_dict() fitness = analysis.calc_fitness(trial_fitness_df) trial_data = {**config, **fitness_vec, 'fitness': fitness, 'trial_index': trial_index} prepath = util.get_prepath(spec, info_space, unit='trial') util.write(trial_data, f'{prepath}_trial_data.json') return trial_data
def analyze_experiment(spec, trial_data_dict): '''Analyze experiment and save data''' info_prepath = spec['meta']['info_prepath'] util.write(trial_data_dict, f'{info_prepath}_trial_data_dict.json') # calculate experiment df experiment_df = calc_experiment_df(trial_data_dict, info_prepath) # plot graph viz.plot_experiment(spec, experiment_df, METRICS_COLS) # zip files predir, _, _, _, _, _ = util.prepath_split(info_prepath) shutil.make_archive(predir, 'zip', predir) logger.info(f'All experiment data zipped to {predir}.zip') return experiment_df
def analyze_session(session_spec, session_df, df_mode, plot=True): '''Analyze session and save data, then return metrics. Note there are 2 types of session_df: body.eval_df and body.train_df''' info_prepath = session_spec['meta']['info_prepath'] session_df = session_df.copy() assert len(session_df) > 2, f'Need more than 2 datapoint to calculate metrics' # first datapoint at frame 0 is empty util.write(session_df, f'{info_prepath}_session_df_{df_mode}.csv') # calculate metrics session_metrics = calc_session_metrics(session_df, ps.get(session_spec, 'env.0.name'), info_prepath, df_mode) if plot: # plot graph viz.plot_session(session_spec, session_metrics, session_df, df_mode) viz.plot_session(session_spec, session_metrics, session_df, df_mode, ma=True) return session_metrics
def save_session_data(session_spec, session_df, session_fig): ''' Save the session data: df, plot. session_df is multi-indexed with (a,e,b), 3 extra levels to read, use: session_df = util.read(filepath, header=[0, 1, 2, 3]) session_data = util.session_df_to_data(session_df) ''' # TODO generalize to use experiment timestamp, id, sesison coor in info space, to replace timestamp spec_name = session_spec['name'] prepath = f'data/{spec_name}/{spec_name}_{util.get_timestamp()}' logger.info(f'Saving session data to {prepath}_*') util.write(session_df, f'{prepath}_session_df.csv') viz.save_image(session_fig, f'{prepath}_session_graph.png')
def calc_experiment_df(trial_data_dict, info_prepath=None): '''Collect all trial data (metrics and config) from trials into a dataframe''' experiment_df = pd.DataFrame(trial_data_dict).transpose() cols = METRICS_COLS config_cols = sorted(ps.difference(experiment_df.columns.tolist(), cols)) sorted_cols = config_cols + cols experiment_df = experiment_df.reindex(sorted_cols, axis=1) experiment_df.sort_values(by=['strength'], ascending=False, inplace=True) if info_prepath is not None: util.write(experiment_df, f'{info_prepath}_experiment_df.csv') # save important metrics in info_prepath directly util.write(experiment_df, f'{info_prepath.replace("info/", "")}_experiment_df.csv') return experiment_df
def run_trial(experiment, config): trial_index = config.pop('trial_index') spec = spec_from_config(experiment, config) info_space = deepcopy(experiment.info_space) info_space.set('trial', trial_index) trial_fitness_df = experiment.init_trial_and_run(spec, info_space) fitness_vec = trial_fitness_df.iloc[0].to_dict() fitness = analysis.calc_fitness(trial_fitness_df) trial_data = { **config, **fitness_vec, 'fitness': fitness, 'trial_index': trial_index, } prepath = analysis.get_prepath(spec, info_space, unit='trial') util.write(trial_data, f'{prepath}_trial_data.json') return trial_data
def save_session_data(spec, info_space, session_data, session_fitness_df, session_fig, body_df_kind='eval'): ''' Save the session data: session_df, session_fitness_df, session_graph. session_data is saved as session_df; multi-indexed with (a,e,b), 3 extra levels to read, use: session_df = util.read(filepath, header=[0, 1, 2, 3], index_col=0) session_data = util.session_df_to_data(session_df) ''' prepath = util.get_prepath(spec, info_space, unit='session') logger.info(f'Saving session data to {prepath}') prefix = 'train' if body_df_kind == 'train' else '' if 'retro_analyze' not in os.environ['PREPATH']: save_session_df(session_data, f'{prepath}_{prefix}session_df.csv', info_space) util.write(session_fitness_df, f'{prepath}_{prefix}session_fitness_df.csv') viz.save_image(session_fig, f'{prepath}_{prefix}session_graph.png')
def analyze_session(session_spec, session_df, df_mode, plot=True): '''Analyze session and save data, then return metrics. Note there are 2 types of session_df: body.eval_df and body.train_df''' info_prepath = session_spec['meta']['info_prepath'] session_df = session_df.copy() # prevent modification assert len(session_df) > 2, f'Need more than 2 datapoint to calculate metrics' # first datapoint at frame 0 is empty util.write(session_df, util.get_session_df_path(session_spec, df_mode)) # calculate metrics session_metrics = calc_session_metrics(session_df, ps.get(session_spec, 'env.0.name'), info_prepath, df_mode) if plot: # plot graph viz.plot_session(session_spec, session_metrics, session_df, df_mode) viz.plot_session(session_spec, session_metrics, session_df, df_mode, ma=True) # manually shut down orca server to avoid zombie processes viz.pio.orca.shutdown_server() return session_metrics
def save_session_data(spec, info_space, session_data, session_fitness_df, session_fig): ''' Save the session data: session_df, session_fitness_df, session_graph. session_data is saved as session_df; multi-indexed with (a,e,b), 3 extra levels to read, use: session_df = util.read(filepath, header=[0, 1, 2, 3]) session_data = util.session_df_to_data(session_df) ''' prepath = util.get_prepath(spec, info_space, unit='session') logger.info(f'Saving session data to {prepath}') if 'retro_analyze' not in os.environ['PREPATH']: session_df = pd.concat(session_data, axis=1) util.write(session_df, f'{prepath}_session_df.csv') util.write(session_fitness_df, f'{prepath}_session_fitness_df.csv') viz.save_image(session_fig, f'{prepath}_session_graph.png')
def analyze_experiment(spec, trial_data_dict): '''Analyze experiment and save data''' info_prepath = spec['meta']['info_prepath'] util.write(trial_data_dict, f'{info_prepath}_trial_data_dict.json') # calculate experiment df experiment_df = calc_experiment_df(trial_data_dict, info_prepath) # plot graph viz.plot_experiment(spec, experiment_df, METRICS_COLS) viz.plot_experiment_trials(spec, experiment_df, METRICS_COLS) # manually shut down orca server to avoid zombie processes viz.pio.orca.shutdown_server() # zip files predir, _, _, _, _, _ = util.prepath_split(info_prepath) zipdir = util.smart_path(predir) shutil.make_archive(zipdir, 'zip', zipdir) logger.info(f'All experiment data zipped to {predir}.zip') return experiment_df
def save_session_df(session_data, prepath, info_space): '''Save session_df, and if is in eval mode, modify it and save with append''' filepath = f'{prepath}_session_df.csv' if util.get_lab_mode() in ('enjoy', 'eval'): ckpt = util.find_ckpt(info_space.eval_model_prepath) epi = int(re.search('epi(\d+)', ckpt)[1]) totalt = int(re.search('totalt(\d+)', ckpt)[1]) session_df = pd.concat(session_data, axis=1) eval_session_df = pd.DataFrame(data=[session_df.mean()]) for aeb in util.get_df_aeb_list(eval_session_df): eval_session_df.loc[:, aeb + ('epi', )] = epi eval_session_df.loc[:, aeb + ('total_t', )] = totalt # if eval, save with append mode header = not os.path.exists(filepath) with open(filepath, 'a') as f: eval_session_df.to_csv(f, header=header) else: session_df = pd.concat(session_data, axis=1) util.write(session_df, filepath)
def save_session_data(spec, info_space, session_mdp_data, session_data, session_fitness_df, session_fig): ''' Save the session data: session_mdp_df, session_df, session_fitness_df, session_graph. session_data is saved as session_df; multi-indexed with (a,e,b), 3 extra levels to read, use: session_df = util.read(filepath, header=[0, 1, 2, 3]) session_data = util.session_df_to_data(session_df) Likewise for session_mdp_df ''' prepath = util.get_prepath(spec, info_space, unit='session') logger.info(f'Saving session data to {prepath}') if session_mdp_data is not None: # not from retro analysis session_mdp_df = pd.concat(session_mdp_data, axis=1) session_df = pd.concat(session_data, axis=1) # TODO reactivate saving when get to the transition matrix research # util.write(session_mdp_df, f'{prepath}_session_mdp_df.csv') util.write(session_df, f'{prepath}_session_df.csv') util.write(session_fitness_df, f'{prepath}_session_fitness_df.csv') viz.save_image(session_fig, f'{prepath}_session_graph.png')
def get_random_baseline(env_name): '''Get a single random baseline for env; if does not exist in file, generate live and update the file''' random_baseline = util.read(FILEPATH) if env_name in random_baseline: baseline = random_baseline[env_name] else: try: logger.info(f'Generating random baseline for {env_name}') baseline = gen_random_baseline(env_name, NUM_EVAL) except Exception as e: logger.warning( f'Cannot start env: {env_name}, skipping random baseline generation' ) baseline = None # update immediately logger.info(f'Updating new random baseline in {FILEPATH}') random_baseline[env_name] = baseline util.write(random_baseline, FILEPATH) return baseline
def generate_specs(spec, const='agent'): ''' Generate benchmark specs with compatible discrete/continuous/both types: - take a spec - for each in benchmark envs - use the template env spec to update spec - append to benchmark specs Interchange agent and env for the reversed benchmark. ''' if const == 'agent': const_name = _.get(spec, 'agent.0.algorithm.name') variant = 'env' else: const_name = _.get(spec, 'env.0.name') variant = 'agent' filepath = f'{spec_util.SPEC_DIR}/benchmark_{const_name}.json' if os.path.exists(filepath): logger.info( f'Benchmark for {const_name} exists at {filepath} already, not overwriting.' ) benchmark_specs = util.read(filepath) return benchmark_specs logger.info(f'Generating benchmark for {const_name}') benchmark_variants = [] benchmark_specs = {} for dist_cont, const_names in BENCHMARK[const].items(): if const_name in const_names: benchmark_variants.extend(BENCHMARK[variant][dist_cont]) for vary_name in benchmark_variants: vary_spec = ENV_TEMPLATES[vary_name] spec_name = f'{const_name}_{vary_name}' benchmark_spec = spec.copy() benchmark_spec['name'] = spec_name benchmark_spec[variant] = [vary_spec] benchmark_specs[spec_name] = benchmark_spec util.write(benchmark_specs, filepath) logger.info(f'Benchmark for {const_name} written to {filepath}.') return benchmark_specs
def save_session_data(spec, info_space, session_mdp_data, session_data, session_fitness_df, session_fig): ''' Save the session data: session_mdp_df, session_df, session_fitness_df, session_graph. session_data is saved as session_df; multi-indexed with (a,e,b), 3 extra levels to read, use: session_df = util.read(filepath, header=[0, 1, 2, 3]) session_data = util.session_df_to_data(session_df) Likewise for session_mdp_df ''' prepath = util.get_prepath(spec, info_space, unit='session') logger.info(f'Saving session data to {prepath}') if session_mdp_data is not None: # not from retro analysis session_mdp_df = pd.concat(session_mdp_data, axis=1) session_df = pd.concat(session_data, axis=1) # TODO reactivate saving when get to the transition matrix research # util.write(session_mdp_df, f'{prepath}_session_mdp_df.csv') util.write(session_df, f'{prepath}_session_df.csv') util.write(session_fitness_df, f'{prepath}_session_fitness_df.csv') # TODO replaced by plot_best_sessions until Feb 2018 https://github.com/plotly/plotly.py/issues/880 if os.environ.get('lab_mode') == 'train': viz.save_image(session_fig, f'{prepath}_session_graph.png')
def save_session_df(session_data, filepath, info_space): '''Save session_df, and if is in eval mode, modify it and save with append''' if util.in_eval_lab_modes(): ckpt = util.find_ckpt(info_space.eval_model_prepath) epi = int(re.search('epi(\d+)', ckpt)[1]) totalt = int(re.search('totalt(\d+)', ckpt)[1]) session_df = pd.concat(session_data, axis=1) mean_sr = session_df.mean() mean_sr.name = totalt # set index to prevent all being the same eval_session_df = pd.DataFrame(data=[mean_sr]) # set sr name too, to total_t for aeb in util.get_df_aeb_list(eval_session_df): eval_session_df.loc[:, aeb + ('epi',)] = epi eval_session_df.loc[:, aeb + ('total_t',)] = totalt # if eval, save with append mode header = not os.path.exists(filepath) with open(filepath, 'a') as f: eval_session_df.to_csv(f, header=header) else: session_df = pd.concat(session_data, axis=1) util.write(session_df, filepath)
def generate_specs(spec, const='agent'): ''' Generate benchmark specs with compatible discrete/continuous/both types: - take a spec - for each in benchmark envs - use the template env spec to update spec - append to benchmark specs Interchange agent and env for the reversed benchmark. ''' if const == 'agent': const_name = ps.get(spec, 'agent.0.algorithm.name') variant = 'env' else: const_name = ps.get(spec, 'env.0.name') variant = 'agent' filepath = f'{spec_util.SPEC_DIR}/benchmark_{const_name}.json' if os.path.exists(filepath): logger.info(f'Benchmark for {const_name} exists at {filepath} already, not overwriting.') benchmark_specs = util.read(filepath) return benchmark_specs logger.info(f'Generating benchmark for {const_name}') benchmark_variants = [] benchmark_specs = {} for dist_cont, const_names in BENCHMARK[const].items(): if const_name in const_names: benchmark_variants.extend(BENCHMARK[variant][dist_cont]) for vary_name in benchmark_variants: vary_spec = ENV_TEMPLATES[vary_name] spec_name = f'{const_name}_{vary_name}' benchmark_spec = spec.copy() benchmark_spec['name'] = spec_name benchmark_spec[variant] = [vary_spec] benchmark_specs[spec_name] = benchmark_spec util.write(benchmark_specs, filepath) logger.info(f'Benchmark for {const_name} written to {filepath}.') return benchmark_specs
def calc_session_metrics(session_df, env_name, info_prepath=None, df_mode=None): ''' Calculate the session metrics: strength, efficiency, stability @param DataFrame:session_df Dataframe containing reward, frame, opt_step @param str:env_name Name of the environment to get its random baseline @param str:info_prepath Optional info_prepath to auto-save the output to @param str:df_mode Optional df_mode to save with info_prepath @returns dict:metrics Consists of scalar metrics and series local metrics ''' rand_bl = random_baseline.get_random_baseline(env_name) mean_rand_returns = rand_bl['mean'] mean_returns = session_df['total_reward'] frames = session_df['frame'] opt_steps = session_df['opt_step'] final_return_ma = mean_returns[-viz.PLOT_MA_WINDOW:].mean() str_, local_strs = calc_strength(mean_returns, mean_rand_returns) max_str, final_str = local_strs.max(), local_strs.iloc[-1] sample_eff, local_sample_effs = calc_efficiency(local_strs, frames) train_eff, local_train_effs = calc_efficiency(local_strs, opt_steps) sta, local_stas = calc_stability(local_strs) # all the scalar session metrics scalar = { 'final_return_ma': final_return_ma, 'strength': str_, 'max_strength': max_str, 'final_strength': final_str, 'sample_efficiency': sample_eff, 'training_efficiency': train_eff, 'stability': sta, } # all the session local metrics local = { 'mean_returns': mean_returns, 'strengths': local_strs, 'sample_efficiencies': local_sample_effs, 'training_efficiencies': local_train_effs, 'stabilities': local_stas, 'frames': frames, 'opt_steps': opt_steps, } metrics = { 'scalar': scalar, 'local': local, } if info_prepath is not None: # auto-save if info_prepath is given util.write(metrics, f'{info_prepath}_session_metrics_{df_mode}.pkl') util.write(scalar, f'{info_prepath}_session_metrics_scalar_{df_mode}.json') # save important metrics in info_prepath directly util.write( scalar, f'{info_prepath.replace("info/", "")}_session_metrics_scalar_{df_mode}.json' ) return metrics
def retro_analyze_trials(predir): '''Retro-analyze all trial level datas.''' logger.info('Retro-analyzing trials from file') from slm_lab.experiment.control import Trial for filename in os.listdir(predir): if filename.endswith('_trial_data.json'): filepath = f'{predir}/{filename}' tn = filename.replace('_trial_data.json', '').split('_')[-1] trial_index = int(tn[1:]) # mock trial spec, info_space = mock_info_space_spec(predir, trial_index) trial = Trial(spec, info_space) session_data_dict = session_data_dict_from_file(predir, trial_index) trial.session_data_dict = session_data_dict trial_fitness_df = analyze_trial(trial) # write trial_data that was written from ray search fitness_vec = trial_fitness_df.iloc[0].to_dict() fitness = calc_fitness(trial_fitness_df) trial_data = util.read(filepath) trial_data.update({ **fitness_vec, 'fitness': fitness, 'trial_index': trial_index, }) util.write(trial_data, filepath)
def calc_trial_metrics(session_metrics_list, info_prepath=None): ''' Calculate the trial metrics: mean(strength), mean(efficiency), mean(stability), consistency @param list:session_metrics_list The metrics collected from each session; format: {session_index: {'scalar': {...}, 'local': {...}}} @param str:info_prepath Optional info_prepath to auto-save the output to @returns dict:metrics Consists of scalar metrics and series local metrics ''' # calculate mean of session metrics scalar_list = [sm['scalar'] for sm in session_metrics_list] mean_scalar = pd.DataFrame(scalar_list).mean().to_dict() mean_returns_list = [sm['local']['mean_returns'] for sm in session_metrics_list] local_strs_list = [sm['local']['strengths'] for sm in session_metrics_list] local_se_list = [sm['local']['sample_efficiencies'] for sm in session_metrics_list] local_te_list = [sm['local']['training_efficiencies'] for sm in session_metrics_list] local_sta_list = [sm['local']['stabilities'] for sm in session_metrics_list] frames = session_metrics_list[0]['local']['frames'] opt_steps = session_metrics_list[0]['local']['opt_steps'] # calculate consistency con, local_cons = calc_consistency(local_strs_list) # all the scalar trial metrics scalar = { 'final_return_ma': mean_scalar['final_return_ma'], 'strength': mean_scalar['strength'], 'max_strength': mean_scalar['max_strength'], 'final_strength': mean_scalar['final_strength'], 'sample_efficiency': mean_scalar['sample_efficiency'], 'training_efficiency': mean_scalar['training_efficiency'], 'stability': mean_scalar['stability'], 'consistency': con, } assert set(scalar.keys()) == set(METRICS_COLS) # for plotting: gather all local series of sessions local = { 'mean_returns': mean_returns_list, 'strengths': local_strs_list, 'sample_efficiencies': local_se_list, 'training_efficiencies': local_te_list, 'stabilities': local_sta_list, 'consistencies': local_cons, # this is a list 'frames': frames, 'opt_steps': opt_steps, } metrics = { 'scalar': scalar, 'local': local, } if info_prepath is not None: # auto-save if info_prepath is given util.write(metrics, f'{info_prepath}_trial_metrics.pkl') util.write(scalar, f'{info_prepath}_trial_metrics_scalar.json') # save important metrics in info_prepath directly util.write(scalar, f'{info_prepath.replace("info/", "")}_trial_metrics_scalar.json') return metrics
def save_session_data(info_space, session_spec, session_mdp_data, session_data, session_fitness_df, session_fig): ''' Save the session data: session_mdp_df, session_df, session_fitness_df, session_graph. session_data is saved as session_df; multi-indexed with (a,e,b), 3 extra levels to read, use: session_df = util.read(filepath, header=[0, 1, 2, 3]) session_data = util.session_df_to_data(session_df) Likewise for session_mdp_df ''' spec_name = session_spec['name'] trial_index = info_space.coor['trial'] session_index = info_space.coor['session'] predir = f'data/{spec_name}_{info_space.experiment_ts}' prename = f'{spec_name}_t{trial_index}_s{session_index}' logger.info( f'Saving trial {trial_index} session {session_index} data to {predir}') session_mdp_df = pd.concat(session_mdp_data, axis=1) session_df = pd.concat(session_data, axis=1) util.write(session_mdp_df, f'{predir}/{prename}_session_mdp_df.csv') util.write(session_df, f'{predir}/{prename}_session_df.csv') util.write(session_fitness_df, f'{predir}/{prename}_session_fitness_df.csv')
def save_trial_data(spec, info_space, trial_fitness_df, trial_fig): '''Save the trial data: spec, trial_fitness_df.''' prepath = util.get_prepath(spec, info_space, unit='trial') logger.info(f'Saving trial data to {prepath}') util.write(trial_fitness_df, f'{prepath}_trial_fitness_df.csv') viz.save_image(trial_fig, f'{prepath}_trial_graph.png')
def test_write_read_as_plain_list(test_str, filename, dtype): data_path = f'test/fixture/common/util/{filename}' util.write(test_str, util.smart_path(data_path)) assert os.path.exists(data_path) data_dict = util.read(util.smart_path(data_path)) assert isinstance(data_dict, dtype)
def test_write_read_as_df(test_df, filename, dtype): data_path = f'test/fixture/common/util/{filename}' util.write(test_df, util.smart_path(data_path)) assert os.path.exists(data_path) data_df = util.read(util.smart_path(data_path)) assert isinstance(data_df, dtype)
def save(spec, unit='experiment'): '''Save spec to proper path. Called at Experiment or Trial init.''' prepath = util.get_prepath(spec, unit) util.write(spec, f'{prepath}_spec.json')
def test_write_read_as_plain_list(test_str, filename, dtype): data_path = f'test/fixture/lib/util/{filename}' util.write(test_str, util.smart_path(data_path)) assert os.path.exists(data_path) data_dict = util.read(util.smart_path(data_path)) assert isinstance(data_dict, dtype)
def test_write_read_as_df(test_df, filename, dtype): data_path = f'test/fixture/lib/util/{filename}' util.write(test_df, util.smart_path(data_path)) assert os.path.exists(data_path) data_df = util.read(util.smart_path(data_path)) assert isinstance(data_df, dtype)
def save_spec(spec, info_space, unit='experiment'): '''Save spec to proper path. Called at Experiment or Trial init.''' prepath = util.get_prepath(spec, info_space, unit) util.write(spec, f'{prepath}_spec.json')
def save_trial_data(spec, info_space, trial_fitness_df): '''Save the trial data: spec, trial_fitness_df.''' prepath = get_prepath(spec, info_space, unit='trial') logger.info(f'Saving trial data to {prepath}') util.write(trial_fitness_df, f'{prepath}_trial_fitness_df.csv')
def save_experiment_data(spec, info_space, experiment_df, experiment_fig): '''Save the experiment data: best_spec, experiment_df, experiment_graph.''' prepath = util.get_prepath(spec, info_space, unit='experiment') logger.info(f'Saving experiment data to {prepath}') util.write(experiment_df, f'{prepath}_experiment_df.csv') viz.save_image(experiment_fig, f'{prepath}_experiment_graph.png')