def get(spec_file, spec_name):
'''
Get an experiment spec from spec_file, spec_name.
Auto-check spec.
@example
spec = spec_util.get('base.json', 'base_case_openai')
'''
spec_dict = util.read(f'{SPEC_DIR}/{spec_file}')
assert spec_name in spec_dict, f'spec_name {spec_name} is not in spec_file {spec_file}. Choose from:\n {ps.join(spec_dict.keys(), ",")}'
spec = spec_dict[spec_name]
spec['name'] = spec_name
spec['git_SHA'] = util.get_git_sha()
check(spec)
return spec
def check_all():
'''Check all spec files, all specs.'''
spec_files = ps.filter_(
os.listdir(SPEC_DIR),
lambda f: f.endswith('.json') and not f.startswith('_'))
for spec_file in spec_files:
spec_dict = util.read(f'{SPEC_DIR}/{spec_file}')
for spec_name, spec in spec_dict.items():
try:
spec['name'] = spec_name
spec['git_SHA'] = util.get_git_sha()
check(spec)
except Exception as e:
logger.exception(f'spec_file {spec_file} fails spec check')
raise e
logger.info(f'Checked all specs from: {ps.join(spec_files, ",")}')
return True
def extend_meta_spec(spec):
'''Extend meta spec with information for lab functions'''
extended_meta_spec = {
# reset lab indices to -1 so that they tick to 0
'experiment': -1,
'trial': -1,
'session': -1,
'cuda_offset': int(os.environ.get('CUDA_OFFSET', 0)),
'experiment_ts': util.get_ts(),
'prepath': None,
# ckpt extends prepath, e.g. ckpt_str = ckpt-epi10-totalt1000
'ckpt': None,
'git_sha': util.get_git_sha(),
'random_seed': None,
'eval_model_prepath': None,
}
spec['meta'].update(extended_meta_spec)
return spec
def get(spec_file, spec_name):
'''
Get an experiment spec from spec_file, spec_name.
Auto-check spec.
@example
spec = spec_util.get('base.json', 'base_case_openai')
'''
if 'data/' in spec_file:
assert spec_name in spec_file, 'spec_file in data/ must be lab-generated and contains spec_name'
spec = util.read(spec_file)
else:
spec_file = f'{SPEC_DIR}/{spec_file}' # allow direct filename
spec_dict = util.read(spec_file)
assert spec_name in spec_dict, f'spec_name {spec_name} is not in spec_file {spec_file}. Choose from:\n {ps.join(spec_dict.keys(), ",")}'
spec = spec_dict[spec_name]
spec['name'] = spec_name
spec['git_SHA'] = util.get_git_sha()
check(spec)
return spec