def check_config(executable, conda_env, configs):
"""Check if the given configs are consistent with the Sacred experiment in the given executable.
Parameters
----------
executable: str
The Python file containing the experiment.
conda_env: str
The experiment's Anaconda environment.
configs: list of dicts
Contains the parameter configurations.
Returns
-------
None
"""
import sacred
exp_module = import_exe(executable, conda_env)
# Extract experiment from module
exps = [v for k, v in exp_module.__dict__.items() if type(v) == sacred.Experiment]
if len(exps) == 0:
raise ExecutableError(f"Found no Sacred experiment. Something is wrong in '{executable}'.")
elif len(exps) > 1:
raise ExecutableError(f"Found more than 1 Sacred experiment in '{executable}'. "
f"Can't check parameter configs. Disable via --no-sanity-check.")
exp = exps[0]
empty_run = sacred.initialize.create_run(exp, exp.default_command, config_updates=None, named_configs=())
captured_args = {
sacred.utils.join_paths(cf.prefix, n)
for cf in exp.captured_functions
for n in cf.signature.arguments
}
for config in configs:
config_added = {k: v for k, v in config.items() if k not in empty_run.config.keys()}
config_flattened = {k for k, _ in sacred.utils.iterate_flattened(config_added)}
# Check for unused arguments
for conf in sorted(config_flattened):
if not (set(sacred.utils.iter_prefixes(conf)) & captured_args):
raise sacred.utils.ConfigAddedError(conf, config=config_added)
# Check for missing arguments
options = empty_run.config.copy()
options.update(config)
options.update({k: None for k in sacred.utils.ConfigAddedError.SPECIAL_ARGS})
empty_run.main_function.signature.construct_arguments((), {}, options, False)
def set_executable_and_working_dir(config_path, seml_dict):
"""
Determine the working directory of the project and chdir into the working directory.
Parameters
----------
config_path: Path to the config file
seml_dict: SEML config dictionary
Returns
-------
None
"""
config_dir = str(Path(config_path).expanduser().resolve().parent)
working_dir = config_dir
os.chdir(working_dir)
if "executable" not in seml_dict:
raise ConfigError(
"Please specify an executable path for the experiment.")
executable = seml_dict['executable']
executable_relative_to_config = os.path.exists(executable)
executable_relative_to_project_root = False
if 'project_root_dir' in seml_dict:
working_dir = str(
Path(seml_dict['project_root_dir']).expanduser().resolve())
seml_dict['use_uploaded_sources'] = True
os.chdir(working_dir) # use project root as base dir from now on
executable_relative_to_project_root = os.path.exists(executable)
del seml_dict[
'project_root_dir'] # from now on we use only the working dir
else:
seml_dict['use_uploaded_sources'] = False
logging.warning(
"'project_root_dir' not defined in seml config. Source files will not be saved in MongoDB."
)
seml_dict['working_dir'] = working_dir
if not (executable_relative_to_config
or executable_relative_to_project_root):
raise ExecutableError(f"Could not find the executable.")
executable = str(Path(executable).expanduser().resolve())
seml_dict['executable'] = (str(Path(executable).relative_to(working_dir))
if executable_relative_to_project_root else str(
Path(executable).relative_to(config_dir)))
def upload_sources(seml_config, collection, batch_id):
root_dir = str(Path(seml_config['working_dir']).expanduser().resolve())
sources = get_imported_sources(seml_config['executable'],
root_dir=root_dir,
conda_env=seml_config['conda_environment'])
executable_abs = str(
Path(seml_config['executable']).expanduser().resolve())
if executable_abs not in sources:
raise ExecutableError(
f"Executable {executable_abs} was not found in the sources to upload."
)
uploaded_files = []
for s in sources:
file_id = upload_file(s, collection, batch_id, 'source_file')
source_path = Path(s)
uploaded_files.append(
(str(source_path.relative_to(root_dir)), file_id))
return uploaded_files