def run_by_mode(spec_file, spec_name, lab_mode):
logger.info(f'Running lab in mode: {lab_mode}')
spec = spec_util.get(spec_file, spec_name)
info_space = InfoSpace()
analysis.save_spec(spec, info_space, unit='experiment')
# '@' is reserved for 'enjoy@{prepath}'
os.environ['lab_mode'] = lab_mode.split('@')[0]
os.environ['PREPATH'] = util.get_prepath(spec, info_space)
reload(logger) # to set PREPATH properly
if lab_mode == 'search':
info_space.tick('experiment')
Experiment(spec, info_space).run()
elif lab_mode.startswith('train'):
if '@' in lab_mode:
prepath = lab_mode.split('@')[1]
spec, info_space = util.prepath_to_spec_info_space(prepath)
else:
info_space.tick('trial')
Trial(spec, info_space).run()
elif lab_mode.startswith('enjoy'):
prepath = lab_mode.split('@')[1]
spec, info_space = util.prepath_to_spec_info_space(prepath)
Session(spec, info_space).run()
elif lab_mode.startswith('enjoy'):
prepath = lab_mode.split('@')[1]
spec, info_space = util.prepath_to_spec_info_space(prepath)
Session(spec, info_space).run()
elif lab_mode == 'dev':
spec = util.override_dev_spec(spec)
info_space.tick('trial')
Trial(spec, info_space).run()
else:
logger.warn('lab_mode not recognized; must be one of `search, train, enjoy, benchmark, dev`.')
def run_by_mode(spec_file, spec_name, lab_mode):
logger.info(f'Running lab in mode: {lab_mode}')
spec = spec_util.get(spec_file, spec_name)
info_space = InfoSpace()
os.environ['PREPATH'] = util.get_prepath(spec, info_space)
reload(logger) # to set PREPATH properly
# expose to runtime, '@' is reserved for 'enjoy@{prepath}'
os.environ['lab_mode'] = lab_mode.split('@')[0]
if lab_mode == 'search':
info_space.tick('experiment')
Experiment(spec, info_space).run()
elif lab_mode == 'train':
info_space.tick('trial')
Trial(spec, info_space).run()
elif lab_mode.startswith('enjoy'):
prepath = lab_mode.split('@')[1]
spec, info_space = util.prepath_to_spec_info_space(prepath)
Session(spec, info_space).run()
elif lab_mode == 'generate_benchmark':
benchmarker.generate_specs(spec, const='agent')
elif lab_mode == 'benchmark':
# TODO allow changing const to env
run_benchmark(spec, const='agent')
elif lab_mode == 'dev':
spec = util.override_dev_spec(spec)
info_space.tick('trial')
Trial(spec, info_space).run()
else:
logger.warn(
'lab_mode not recognized; must be one of `search, train, enjoy, benchmark, dev`.'
)
def retro_analyze_trials(predir):
'''Retro-analyze all trial level datas.'''
logger.info('Retro-analyzing trials from file')
from slm_lab.experiment.control import Trial
filenames = ps.filter_(os.listdir(predir),
lambda filename: filename.endswith('_trial_df.csv'))
for idx, filename in enumerate(filenames):
filepath = f'{predir}/{filename}'
prepath = filepath.replace('_trial_df.csv', '')
spec, info_space = util.prepath_to_spec_info_space(prepath)
trial_index, _ = util.prepath_to_idxs(prepath)
trial = Trial(spec, info_space)
trial.session_data_dict = session_data_dict_from_file(
predir, trial_index, ps.get(info_space, 'ckpt'))
# zip only at the last
zip = (idx == len(filenames) - 1)
trial_fitness_df = analysis.analyze_trial(trial, zip)
# write trial_data that was written from ray search
trial_data_filepath = filepath.replace('_trial_df.csv',
'_trial_data.json')
if os.path.exists(trial_data_filepath):
fitness_vec = trial_fitness_df.iloc[0].to_dict()
fitness = analysis.calc_fitness(trial_fitness_df)
trial_data = util.read(trial_data_filepath)
trial_data.update({
**fitness_vec,
'fitness': fitness,
'trial_index': trial_index,
})
util.write(trial_data, trial_data_filepath)
def retro_analyze_sessions(predir):
'''Retro-analyze all session level datas.'''
logger.info('Retro-analyzing sessions from file')
from slm_lab.experiment.control import Session, SpaceSession
for filename in os.listdir(predir):
# to account for both types of session_df
if filename.endswith('_session_df.csv'):
body_df_kind = 'eval' # from body.eval_df
prefix = ''
is_session_df = True
elif filename.endswith('_trainsession_df.csv'):
body_df_kind = 'train' # from body.train_df
prefix = 'train'
is_session_df = True
else:
is_session_df = False
if is_session_df:
prepath = f'{predir}/{filename}'.replace(
f'_{prefix}session_df.csv', '')
spec, info_space = util.prepath_to_spec_info_space(prepath)
trial_index, session_index = util.prepath_to_idxs(prepath)
SessionClass = Session if spec_util.is_singleton(
spec) else SpaceSession
session = SessionClass(spec, info_space)
session_data = session_data_from_file(predir, trial_index,
session_index,
ps.get(info_space,
'ckpt'), prefix)
analysis._analyze_session(session, session_data, body_df_kind)
def retro_analyze_experiment(predir):
'''Retro-analyze all experiment level datas.'''
logger.info('Retro-analyzing experiment from file')
from slm_lab.experiment.control import Experiment
_, _, _, spec_name, _, _ = util.prepath_split(predir)
prepath = f'{predir}/{spec_name}'
spec, info_space = util.prepath_to_spec_info_space(prepath)
experiment = Experiment(spec, info_space)
experiment.trial_data_dict = trial_data_dict_from_file(predir)
return analyze_experiment(experiment)
def retro_analyze_sessions(predir):
'''Retro-analyze all session level datas.'''
logger.info('Retro-analyzing sessions from file')
from slm_lab.experiment.control import Session, SpaceSession
for filename in os.listdir(predir):
if filename.endswith('_session_df.csv'):
prepath = f'{predir}/{filename}'.replace('_session_df.csv', '')
spec, info_space = util.prepath_to_spec_info_space(prepath)
trial_index, session_index = util.prepath_to_idxs(prepath)
SessionClass = Session if spec_util.is_singleton(
spec) else SpaceSession
session = SessionClass(spec, info_space)
session_data = session_data_from_file(predir, trial_index,
session_index)
analyze_session(session, session_data)
def analyze_session(session, eager_analyze_trial=False, tmp_space_session_sub=False):
'''
Gather session data, plot, and return fitness df for high level agg.
@returns {DataFrame} session_fitness_df Single-row df of session fitness vector (avg over aeb), indexed with session index.
'''
logger.info('Analyzing session')
session_data = get_session_data(session, body_df_kind='train')
session_fitness_df = _analyze_session(session, session_data, body_df_kind='train')
session_data = get_session_data(session, body_df_kind='eval', tmp_space_session_sub=tmp_space_session_sub)
session_fitness_df = _analyze_session(session, session_data, body_df_kind='eval')
if eager_analyze_trial:
# for live trial graph, analyze trial after analyzing session, this only takes a second
from slm_lab.experiment import retro_analysis
prepath = util.get_prepath(session.spec, session.info_space, unit='session')
# use new ones to prevent side effects
spec, info_space = util.prepath_to_spec_info_space(prepath)
predir, _, _, _, _, _ = util.prepath_split(prepath)
retro_analysis.analyze_eval_trial(spec, info_space, predir)
return session_fitness_df
def run_old_mode(spec_file, spec_name, lab_mode):
'''Run using existing data with `enjoy, eval`. The eval mode is also what train mode's online eval runs in a subprocess via bash command'''
# reconstruct spec and info_space from existing data
lab_mode, prename = lab_mode.split('@')
predir, _, _, _, _, _ = util.prepath_split(spec_file)
prepath = f'{predir}/{prename}'
spec, info_space = util.prepath_to_spec_info_space(prepath)
# see InfoSpace def for more on these
info_space.ckpt = 'eval'
info_space.eval_model_prepath = prepath
# no info_space.tick() as they are reconstructed
if lab_mode == 'enjoy':
spec = spec_util.override_enjoy_spec(spec)
Session(spec, info_space).run()
elif lab_mode == 'eval':
spec = spec_util.override_eval_spec(spec)
Session(spec, info_space).run()
util.clear_periodic_ckpt(prepath) # cleanup after itself
analysis.analyze_eval_trial(spec, info_space, predir)
else:
raise ValueError('Unrecognizable lab_mode not of `enjoy, eval`')
def run_old_mode(spec_file, spec_name, lab_mode):
'''Run using existing data with `enjoy, eval`. The eval mode is also what train mode's online eval runs in a subprocess via bash command'''
# reconstruct spec and info_space from existing data
lab_mode, prename = lab_mode.split('@')
predir, _, _, _, _, _ = util.prepath_split(spec_file)
prepath = f'{predir}/{prename}'
spec, info_space = util.prepath_to_spec_info_space(prepath)
# see InfoSpace def for more on these
info_space.ckpt = 'eval'
info_space.eval_model_prepath = prepath
# no info_space.tick() as they are reconstructed
if lab_mode == 'enjoy':
spec = spec_util.override_enjoy_spec(spec)
Session(spec, info_space).run()
elif lab_mode == 'eval':
# example eval command:
# python run_lab.py data/dqn_cartpole_2018_12_19_224811/dqn_cartpole_t0_spec.json dqn_cartpole eval@dqn_cartpole_t0_s1_ckpt-epi10-totalt1000
spec = spec_util.override_eval_spec(spec)
Session(spec, info_space).run()
util.clear_periodic_ckpt(prepath) # cleanup after itself
retro_analysis.analyze_eval_trial(spec, info_space, predir)
else:
raise ValueError(f'Unrecognizable lab_mode not of {EVAL_MODES}')
def retro_analyze_trials(predir):
'''Retro-analyze all trial level datas.'''
logger.info('Retro-analyzing trials from file')
from slm_lab.experiment.control import Trial
for filename in os.listdir(predir):
if filename.endswith('_trial_data.json'):
filepath = f'{predir}/{filename}'
prepath = filepath.replace('_trial_data.json', '')
spec, info_space = util.prepath_to_spec_info_space(prepath)
trial_index, _ = util.prepath_to_idxs(prepath)
trial = Trial(spec, info_space)
trial.session_data_dict = session_data_dict_from_file(
predir, trial_index)
trial_fitness_df = analyze_trial(trial)
# write trial_data that was written from ray search
fitness_vec = trial_fitness_df.iloc[0].to_dict()
fitness = calc_fitness(trial_fitness_df)
trial_data = util.read(filepath)
trial_data.update({
**fitness_vec,
'fitness': fitness,
'trial_index': trial_index,
})
util.write(trial_data, filepath)
def run_online_eval_from_prepath(prepath):
'''Used by retro_eval'''
spec, info_space = util.prepath_to_spec_info_space(prepath)
ckpt = util.find_ckpt(prepath)
return run_online_eval(spec, info_space, ckpt)