def run_by_mode(spec_file, spec_name, lab_mode): logger.info(f'Running lab in mode: {lab_mode}') spec = spec_util.get(spec_file, spec_name) info_space = InfoSpace() analysis.save_spec(spec, info_space, unit='experiment') # '@' is reserved for 'enjoy@{prepath}' os.environ['lab_mode'] = lab_mode.split('@')[0] os.environ['PREPATH'] = util.get_prepath(spec, info_space) reload(logger) # to set PREPATH properly if lab_mode == 'search': info_space.tick('experiment') Experiment(spec, info_space).run() elif lab_mode.startswith('train'): if '@' in lab_mode: prepath = lab_mode.split('@')[1] spec, info_space = util.prepath_to_spec_info_space(prepath) else: info_space.tick('trial') Trial(spec, info_space).run() elif lab_mode.startswith('enjoy'): prepath = lab_mode.split('@')[1] spec, info_space = util.prepath_to_spec_info_space(prepath) Session(spec, info_space).run() elif lab_mode.startswith('enjoy'): prepath = lab_mode.split('@')[1] spec, info_space = util.prepath_to_spec_info_space(prepath) Session(spec, info_space).run() elif lab_mode == 'dev': spec = util.override_dev_spec(spec) info_space.tick('trial') Trial(spec, info_space).run() else: logger.warn('lab_mode not recognized; must be one of `search, train, enjoy, benchmark, dev`.')
def run_by_mode(spec_file, spec_name, lab_mode): logger.info(f'Running lab in mode: {lab_mode}') spec = spec_util.get(spec_file, spec_name) info_space = InfoSpace() os.environ['PREPATH'] = util.get_prepath(spec, info_space) reload(logger) # to set PREPATH properly # expose to runtime, '@' is reserved for 'enjoy@{prepath}' os.environ['lab_mode'] = lab_mode.split('@')[0] if lab_mode == 'search': info_space.tick('experiment') Experiment(spec, info_space).run() elif lab_mode == 'train': info_space.tick('trial') Trial(spec, info_space).run() elif lab_mode.startswith('enjoy'): prepath = lab_mode.split('@')[1] spec, info_space = util.prepath_to_spec_info_space(prepath) Session(spec, info_space).run() elif lab_mode == 'generate_benchmark': benchmarker.generate_specs(spec, const='agent') elif lab_mode == 'benchmark': # TODO allow changing const to env run_benchmark(spec, const='agent') elif lab_mode == 'dev': spec = util.override_dev_spec(spec) info_space.tick('trial') Trial(spec, info_space).run() else: logger.warn( 'lab_mode not recognized; must be one of `search, train, enjoy, benchmark, dev`.' )
def retro_analyze_trials(predir): '''Retro-analyze all trial level datas.''' logger.info('Retro-analyzing trials from file') from slm_lab.experiment.control import Trial filenames = ps.filter_(os.listdir(predir), lambda filename: filename.endswith('_trial_df.csv')) for idx, filename in enumerate(filenames): filepath = f'{predir}/{filename}' prepath = filepath.replace('_trial_df.csv', '') spec, info_space = util.prepath_to_spec_info_space(prepath) trial_index, _ = util.prepath_to_idxs(prepath) trial = Trial(spec, info_space) trial.session_data_dict = session_data_dict_from_file( predir, trial_index, ps.get(info_space, 'ckpt')) # zip only at the last zip = (idx == len(filenames) - 1) trial_fitness_df = analysis.analyze_trial(trial, zip) # write trial_data that was written from ray search trial_data_filepath = filepath.replace('_trial_df.csv', '_trial_data.json') if os.path.exists(trial_data_filepath): fitness_vec = trial_fitness_df.iloc[0].to_dict() fitness = analysis.calc_fitness(trial_fitness_df) trial_data = util.read(trial_data_filepath) trial_data.update({ **fitness_vec, 'fitness': fitness, 'trial_index': trial_index, }) util.write(trial_data, trial_data_filepath)
def retro_analyze_sessions(predir): '''Retro-analyze all session level datas.''' logger.info('Retro-analyzing sessions from file') from slm_lab.experiment.control import Session, SpaceSession for filename in os.listdir(predir): # to account for both types of session_df if filename.endswith('_session_df.csv'): body_df_kind = 'eval' # from body.eval_df prefix = '' is_session_df = True elif filename.endswith('_trainsession_df.csv'): body_df_kind = 'train' # from body.train_df prefix = 'train' is_session_df = True else: is_session_df = False if is_session_df: prepath = f'{predir}/{filename}'.replace( f'_{prefix}session_df.csv', '') spec, info_space = util.prepath_to_spec_info_space(prepath) trial_index, session_index = util.prepath_to_idxs(prepath) SessionClass = Session if spec_util.is_singleton( spec) else SpaceSession session = SessionClass(spec, info_space) session_data = session_data_from_file(predir, trial_index, session_index, ps.get(info_space, 'ckpt'), prefix) analysis._analyze_session(session, session_data, body_df_kind)
def retro_analyze_experiment(predir): '''Retro-analyze all experiment level datas.''' logger.info('Retro-analyzing experiment from file') from slm_lab.experiment.control import Experiment _, _, _, spec_name, _, _ = util.prepath_split(predir) prepath = f'{predir}/{spec_name}' spec, info_space = util.prepath_to_spec_info_space(prepath) experiment = Experiment(spec, info_space) experiment.trial_data_dict = trial_data_dict_from_file(predir) return analyze_experiment(experiment)
def retro_analyze_sessions(predir): '''Retro-analyze all session level datas.''' logger.info('Retro-analyzing sessions from file') from slm_lab.experiment.control import Session, SpaceSession for filename in os.listdir(predir): if filename.endswith('_session_df.csv'): prepath = f'{predir}/{filename}'.replace('_session_df.csv', '') spec, info_space = util.prepath_to_spec_info_space(prepath) trial_index, session_index = util.prepath_to_idxs(prepath) SessionClass = Session if spec_util.is_singleton( spec) else SpaceSession session = SessionClass(spec, info_space) session_data = session_data_from_file(predir, trial_index, session_index) analyze_session(session, session_data)
def analyze_session(session, eager_analyze_trial=False, tmp_space_session_sub=False): ''' Gather session data, plot, and return fitness df for high level agg. @returns {DataFrame} session_fitness_df Single-row df of session fitness vector (avg over aeb), indexed with session index. ''' logger.info('Analyzing session') session_data = get_session_data(session, body_df_kind='train') session_fitness_df = _analyze_session(session, session_data, body_df_kind='train') session_data = get_session_data(session, body_df_kind='eval', tmp_space_session_sub=tmp_space_session_sub) session_fitness_df = _analyze_session(session, session_data, body_df_kind='eval') if eager_analyze_trial: # for live trial graph, analyze trial after analyzing session, this only takes a second from slm_lab.experiment import retro_analysis prepath = util.get_prepath(session.spec, session.info_space, unit='session') # use new ones to prevent side effects spec, info_space = util.prepath_to_spec_info_space(prepath) predir, _, _, _, _, _ = util.prepath_split(prepath) retro_analysis.analyze_eval_trial(spec, info_space, predir) return session_fitness_df
def run_old_mode(spec_file, spec_name, lab_mode): '''Run using existing data with `enjoy, eval`. The eval mode is also what train mode's online eval runs in a subprocess via bash command''' # reconstruct spec and info_space from existing data lab_mode, prename = lab_mode.split('@') predir, _, _, _, _, _ = util.prepath_split(spec_file) prepath = f'{predir}/{prename}' spec, info_space = util.prepath_to_spec_info_space(prepath) # see InfoSpace def for more on these info_space.ckpt = 'eval' info_space.eval_model_prepath = prepath # no info_space.tick() as they are reconstructed if lab_mode == 'enjoy': spec = spec_util.override_enjoy_spec(spec) Session(spec, info_space).run() elif lab_mode == 'eval': spec = spec_util.override_eval_spec(spec) Session(spec, info_space).run() util.clear_periodic_ckpt(prepath) # cleanup after itself analysis.analyze_eval_trial(spec, info_space, predir) else: raise ValueError('Unrecognizable lab_mode not of `enjoy, eval`')
def run_old_mode(spec_file, spec_name, lab_mode): '''Run using existing data with `enjoy, eval`. The eval mode is also what train mode's online eval runs in a subprocess via bash command''' # reconstruct spec and info_space from existing data lab_mode, prename = lab_mode.split('@') predir, _, _, _, _, _ = util.prepath_split(spec_file) prepath = f'{predir}/{prename}' spec, info_space = util.prepath_to_spec_info_space(prepath) # see InfoSpace def for more on these info_space.ckpt = 'eval' info_space.eval_model_prepath = prepath # no info_space.tick() as they are reconstructed if lab_mode == 'enjoy': spec = spec_util.override_enjoy_spec(spec) Session(spec, info_space).run() elif lab_mode == 'eval': # example eval command: # python run_lab.py data/dqn_cartpole_2018_12_19_224811/dqn_cartpole_t0_spec.json dqn_cartpole eval@dqn_cartpole_t0_s1_ckpt-epi10-totalt1000 spec = spec_util.override_eval_spec(spec) Session(spec, info_space).run() util.clear_periodic_ckpt(prepath) # cleanup after itself retro_analysis.analyze_eval_trial(spec, info_space, predir) else: raise ValueError(f'Unrecognizable lab_mode not of {EVAL_MODES}')
def retro_analyze_trials(predir): '''Retro-analyze all trial level datas.''' logger.info('Retro-analyzing trials from file') from slm_lab.experiment.control import Trial for filename in os.listdir(predir): if filename.endswith('_trial_data.json'): filepath = f'{predir}/{filename}' prepath = filepath.replace('_trial_data.json', '') spec, info_space = util.prepath_to_spec_info_space(prepath) trial_index, _ = util.prepath_to_idxs(prepath) trial = Trial(spec, info_space) trial.session_data_dict = session_data_dict_from_file( predir, trial_index) trial_fitness_df = analyze_trial(trial) # write trial_data that was written from ray search fitness_vec = trial_fitness_df.iloc[0].to_dict() fitness = calc_fitness(trial_fitness_df) trial_data = util.read(filepath) trial_data.update({ **fitness_vec, 'fitness': fitness, 'trial_index': trial_index, }) util.write(trial_data, filepath)
def run_online_eval_from_prepath(prepath): '''Used by retro_eval''' spec, info_space = util.prepath_to_spec_info_space(prepath) ckpt = util.find_ckpt(prepath) return run_online_eval(spec, info_space, ckpt)