def run_trial_test(spec_file, spec_name=False):
spec = spec_util.get(spec_file, spec_name)
spec = util.override_test_spec(spec)
info_space = InfoSpace()
info_space.tick('trial')
trial = Trial(spec, info_space)
trial_data = trial.run()
assert isinstance(trial_data, pd.DataFrame)
def run_benchmark(spec, const):
benchmark_specs = benchmarker.generate_specs(spec, const)
logger.info('Running benchmark')
for spec_name, benchmark_spec in benchmark_specs.items():
# run only if not already exist; benchmark mode only
if not any(spec_name in filename for filename in os.listdir('data')):
info_space = InfoSpace()
info_space.tick('experiment')
Experiment(benchmark_spec, info_space).run()
def run_trial_test(spec_file, spec_name=False, distributed=False):
spec = spec_util.get(spec_file, spec_name)
spec = util.override_test_spec(spec)
info_space = InfoSpace()
info_space.tick('trial')
if distributed:
spec['meta']['distributed'] = True
if os.environ.get('CI') != 'true': # CI has not enough CPU
spec['meta']['max_session'] = 2
trial = Trial(spec, info_space)
trial_data = trial.run()
assert isinstance(trial_data, pd.DataFrame)
def run_trial_test_dist(spec_file, spec_name=False):
spec = spec_util.get(spec_file, spec_name)
spec = spec_util.override_test_spec(spec)
info_space = InfoSpace()
info_space.tick('trial')
spec['meta']['distributed'] = True
spec['meta']['max_session'] = 2
trial = Trial(spec, info_space)
# manually run the logic to obtain global nets for testing to ensure global net gets updated
global_nets = trial.init_global_nets()
# only test first network
if ps.is_list(global_nets): # multiagent only test first
net = list(global_nets[0].values())[0]
else:
net = list(global_nets.values())[0]
session_datas = trial.parallelize_sessions(global_nets)
trial.session_data_dict = {data.index[0]: data for data in session_datas}
trial_data = analysis.analyze_trial(trial)
trial.close()
assert isinstance(trial_data, pd.DataFrame)
def run_by_mode(spec_file, spec_name, lab_mode):
logger.info(f'Running lab in mode: {lab_mode}')
spec = spec_util.get(spec_file, spec_name)
info_space = InfoSpace()
os.environ['PREPATH'] = util.get_prepath(spec, info_space)
reload(logger) # to set PREPATH properly
# expose to runtime, '@' is reserved for 'enjoy@{prepath}'
os.environ['lab_mode'] = lab_mode.split('@')[0]
if lab_mode == 'search':
info_space.tick('experiment')
Experiment(spec, info_space).run()
elif lab_mode == 'train':
info_space.tick('trial')
Trial(spec, info_space).run()
elif lab_mode.startswith('enjoy'):
prepath = lab_mode.split('@')[1]
spec, info_space = util.prepath_to_spec_info_space(prepath)
Session(spec, info_space).run()
elif lab_mode == 'generate_benchmark':
benchmarker.generate_specs(spec, const='agent')
elif lab_mode == 'benchmark':
# TODO allow changing const to env
run_benchmark(spec, const='agent')
elif lab_mode == 'dev':
spec = util.override_dev_spec(spec)
info_space.tick('trial')
Trial(spec, info_space).run()
else:
logger.warn(
'lab_mode not recognized; must be one of `search, train, enjoy, benchmark, dev`.'
)
def run_by_mode(spec_file, spec_name, lab_mode):
logger.info(f'Running lab in mode: {lab_mode}')
spec = spec_util.get(spec_file, spec_name)
info_space = InfoSpace()
analysis.save_spec(spec, info_space, unit='experiment')
# '@' is reserved for 'enjoy@{prepath}'
os.environ['lab_mode'] = lab_mode.split('@')[0]
os.environ['PREPATH'] = util.get_prepath(spec, info_space)
reload(logger) # to set PREPATH properly
if lab_mode == 'search':
info_space.tick('experiment')
Experiment(spec, info_space).run()
elif lab_mode.startswith('train'):
if '@' in lab_mode:
prepath = lab_mode.split('@')[1]
spec, info_space = util.prepath_to_spec_info_space(prepath)
else:
info_space.tick('trial')
Trial(spec, info_space).run()
elif lab_mode.startswith('enjoy'):
prepath = lab_mode.split('@')[1]
spec, info_space = util.prepath_to_spec_info_space(prepath)
Session(spec, info_space).run()
elif lab_mode.startswith('enjoy'):
prepath = lab_mode.split('@')[1]
spec, info_space = util.prepath_to_spec_info_space(prepath)
Session(spec, info_space).run()
elif lab_mode == 'dev':
spec = util.override_dev_spec(spec)
info_space.tick('trial')
Trial(spec, info_space).run()
else:
logger.warn('lab_mode not recognized; must be one of `search, train, enjoy, benchmark, dev`.')
def run_new_mode(spec_file, spec_name, lab_mode):
'''Run to generate new data with `search, train, dev`'''
spec = spec_util.get(spec_file, spec_name)
info_space = InfoSpace()
analysis.save_spec(spec, info_space, unit='experiment') # first save the new spec
if lab_mode == 'search':
info_space.tick('experiment')
Experiment(spec, info_space).run()
elif lab_mode.startswith('train'):
info_space.tick('trial')
Trial(spec, info_space).run()
elif lab_mode == 'dev':
spec = spec_util.override_dev_spec(spec)
info_space.tick('trial')
Trial(spec, info_space).run()
else:
raise ValueError(f'Unrecognizable lab_mode not of {TRAIN_MODES}')