def read_config(config_path):
with open(config_path, 'r') as conf:
config_dict = convert_values(yaml.load(conf, Loader=YamlUniqueLoader))
if "seml" not in config_dict:
raise ConfigError("Please specify a 'seml' dictionary.")
seml_dict = config_dict['seml']
del config_dict['seml']
for k in seml_dict.keys():
if k not in SETTINGS.VALID_SEML_CONFIG_VALUES:
raise ConfigError(
f"{k} is not a valid value in the `seml` config block.")
set_executable_and_working_dir(config_path, seml_dict)
if 'output_dir' in seml_dict:
seml_dict['output_dir'] = str(
Path(seml_dict['output_dir']).expanduser().resolve())
if 'slurm' in config_dict:
slurm_dict = config_dict['slurm']
del config_dict['slurm']
for k in slurm_dict.keys():
if k not in SETTINGS.VALID_SLURM_CONFIG_VALUES:
raise ConfigError(
f"{k} is not a valid value in the `slurm` config block.")
return seml_dict, slurm_dict, config_dict
else:
return seml_dict, None, config_dict
def detect_duplicate_parameters(inverted_config: dict,
sub_config_name: str = None,
ignore_keys: dict = None):
if ignore_keys is None:
ignore_keys = {'random': ('seed', 'samples')}
duplicate_keys = []
for p, l in inverted_config.items():
if len(l) > 1:
if 'random' in l and p in ignore_keys['random']:
continue
duplicate_keys.append((p, l))
if len(duplicate_keys) > 0:
if sub_config_name:
raise ConfigError(
f"Found duplicate keys in sub-config {sub_config_name}: "
f"{duplicate_keys}")
else:
raise ConfigError(f"Found duplicate keys: {duplicate_keys}")
start_characters = set([x[0] for x in inverted_config.keys()])
buckets = {
k: {x
for x in inverted_config.keys() if x.startswith(k)}
for k in start_characters
}
if sub_config_name:
error_str = (
f"Conflicting parameters in sub-config {sub_config_name}, most likely "
"due to ambiguous use of dot-notation in the config dict. Found "
"parameter '{p1}' in dot-notation starting with other parameter "
"'{p2}', which is ambiguous.")
else:
error_str = (
f"Conflicting parameters, most likely "
"due to ambiguous use of dot-notation in the config dict. Found "
"parameter '{p1}' in dot-notation starting with other parameter "
"'{p2}', which is ambiguous.")
for k in buckets.keys():
for p1, p2 in combinations(buckets[k], r=2):
if p1.startswith(
f"{p2}."
): # with "." after p2 to catch cases like "test" and "test1", which are valid.
raise ConfigError(error_str.format(p1=p1, p2=p2))
elif p2.startswith(f"{p1}."):
raise ConfigError(error_str.format(p1=p1, p2=p2))
def set_slurm_job_name(sbatch_options, name, exp):
if 'job-name' in sbatch_options:
raise ConfigError(
"Can't set sbatch `job-name` parameter explicitly. "
"Use `name` parameter instead and SEML will do that for you.")
job_name = f"{name}_{exp['batch_id']}"
sbatch_options['job-name'] = job_name
def convert_parameter_collections(input_config: dict):
flattened_dict = flatten(input_config)
parameter_collection_keys = [
k for k in flattened_dict.keys()
if flattened_dict[k] == "parameter_collection"
]
if len(parameter_collection_keys) > 0:
logging.warning(
"Parameter collections are deprecated. Use dot-notation for nested parameters instead."
)
while len(parameter_collection_keys) > 0:
k = parameter_collection_keys[0]
del flattened_dict[k]
# sub1.sub2.type ==> # sub1.sub2
k = ".".join(k.split(".")[:-1])
parameter_collections_params = [
param_key for param_key in flattened_dict.keys()
if param_key.startswith(k)
]
for p in parameter_collections_params:
if f"{k}.params" in p:
new_key = p.replace(f"{k}.params", k)
if new_key in flattened_dict:
raise ConfigError(
f"Could not convert parameter collections due to key collision: {new_key}."
)
flattened_dict[new_key] = flattened_dict[p]
del flattened_dict[p]
parameter_collection_keys = [
k for k in flattened_dict.keys()
if flattened_dict[k] == "parameter_collection"
]
return unflatten(flattened_dict)
def construct_mapping(loader, node, deep=False):
"""Construct a YAML mapping node, avoiding duplicates"""
loader.flatten_mapping(node)
result = {}
for key_node, value_node in node.value:
key = loader.construct_object(key_node, deep=deep)
if key in result:
raise ConfigError(f"Found duplicate keys: '{key}'")
result[key] = loader.construct_object(value_node, deep=deep)
return result
def get_output_dir_path(config):
if 'output_dir' in config['slurm']:
logging.warning("'output_dir' has moved from 'slurm' to 'seml'. Please adapt your YAML accordingly"
"by moving the 'output_dir' parameter from 'slurm' to 'seml'.")
output_dir = config['slurm']['output_dir']
elif 'output_dir' in config['seml']:
output_dir = config['seml']['output_dir']
else:
output_dir = '.'
output_dir_path = str(Path(output_dir).expanduser().resolve())
if not os.path.isdir(output_dir_path):
raise ConfigError(f"Output directory '{output_dir_path}' does not exist.")
return output_dir_path
def unpack_config(config):
config = convert_parameter_collections(config)
children = {}
reserved_dict = {}
for key, value in config.items():
if not isinstance(value, dict):
continue
if key not in RESERVED_KEYS:
children[key] = value
else:
if key == 'random':
if 'samples' not in value:
raise ConfigError('Random parameters must specify "samples", i.e. the number of random samples.')
reserved_dict[key] = value
else:
reserved_dict[key] = value
return reserved_dict, children
def set_executable_and_working_dir(config_path, seml_dict):
"""
Determine the working directory of the project and chdir into the working directory.
Parameters
----------
config_path: Path to the config file
seml_dict: SEML config dictionary
Returns
-------
None
"""
config_dir = str(Path(config_path).expanduser().resolve().parent)
working_dir = config_dir
os.chdir(working_dir)
if "executable" not in seml_dict:
raise ConfigError(
"Please specify an executable path for the experiment.")
executable = seml_dict['executable']
executable_relative_to_config = os.path.exists(executable)
executable_relative_to_project_root = False
if 'project_root_dir' in seml_dict:
working_dir = str(
Path(seml_dict['project_root_dir']).expanduser().resolve())
seml_dict['use_uploaded_sources'] = True
os.chdir(working_dir) # use project root as base dir from now on
executable_relative_to_project_root = os.path.exists(executable)
del seml_dict[
'project_root_dir'] # from now on we use only the working dir
else:
seml_dict['use_uploaded_sources'] = False
logging.warning(
"'project_root_dir' not defined in seml config. Source files will not be saved in MongoDB."
)
seml_dict['working_dir'] = working_dir
if not (executable_relative_to_config
or executable_relative_to_project_root):
raise ExecutableError(f"Could not find the executable.")
executable = str(Path(executable).expanduser().resolve())
seml_dict['executable'] = (str(Path(executable).relative_to(working_dir))
if executable_relative_to_project_root else str(
Path(executable).relative_to(config_dir)))
def start_sbatch_job(collection, exp_array, unobserved=False, name=None,
output_dir_path=".", sbatch_options=None, max_jobs_per_batch=None,
debug_server=False):
"""Run a list of experiments as a job on the Slurm cluster.
Parameters
----------
collection: pymongo.collection.Collection
The MongoDB collection containing the experiments.
exp_array: List[List[dict]]
List of chunks of experiments to run. Each chunk is a list of experiments.
unobserved: bool
Disable all Sacred observers (nothing written to MongoDB).
name: str
Job name, used by Slurm job and output file.
output_dir_path: str
Directory (relative to home directory) where to store the slurm output files.
sbatch_options: dict
A dictionary that contains options for #SBATCH, e.g. {'mem': 8000} to limit the job's memory to 8,000 MB.
max_jobs_per_batch: int
Maximum number of Slurm jobs running per experiment batch.
debug_server: bool
Run jobs with a debug server.
Returns
-------
None
"""
# Set Slurm job array options
sbatch_options['array'] = f"0-{len(exp_array) - 1}"
if max_jobs_per_batch is not None:
sbatch_options['array'] += f"%{max_jobs_per_batch}"
# Set Slurm output parameter
if 'output' in sbatch_options:
raise ConfigError(f"Can't set sbatch `output` Parameter explicitly. SEML will do that for you.")
elif output_dir_path == "/dev/null":
output_file = output_dir_path
else:
output_file = f'{output_dir_path}/{name}_%A_%a.out'
sbatch_options['output'] = output_file
# Construct sbatch options string
sbatch_options_str = create_slurm_options_string(sbatch_options, False)
# Construct chunked list with all experiment IDs
expid_strings = [('"' + ';'.join([str(exp['_id']) for exp in chunk]) + '"') for chunk in exp_array]
with_sources = ('source_files' in exp_array[0][0]['seml'])
use_conda_env = ('conda_environment' in exp_array[0][0]['seml']
and exp_array[0][0]['seml']['conda_environment'] is not None)
# Construct Slurm script
template = pkg_resources.resource_string(__name__, "slurm_template.sh").decode("utf-8")
prepare_experiment_script = pkg_resources.resource_string(__name__, "prepare_experiment.py").decode("utf-8")
prepare_experiment_script = prepare_experiment_script.replace("'", "'\\''")
if 'working_dir' in exp_array[0][0]['seml']:
working_dir = exp_array[0][0]['seml']['working_dir']
else:
working_dir = "${{SLURM_SUBMIT_DIR}}"
script = template.format(
sbatch_options=sbatch_options_str,
working_dir=working_dir,
use_conda_env=str(use_conda_env).lower(),
conda_env=exp_array[0][0]['seml']['conda_environment'] if use_conda_env else "",
exp_ids=' '.join(expid_strings),
with_sources=str(with_sources).lower(),
prepare_experiment_script=prepare_experiment_script,
db_collection_name=collection.name,
sources_argument="--stored-sources-dir $tmpdir" if with_sources else "",
verbose=logging.root.level <= logging.VERBOSE,
unobserved=unobserved,
debug_server=debug_server,
)
random_int = np.random.randint(0, 999999)
path = f"/tmp/{random_int}.sh"
while os.path.exists(path):
random_int = np.random.randint(0, 999999)
path = f"/tmp/{random_int}.sh"
with open(path, "w") as f:
f.write(script)
output = subprocess.run(f'sbatch {path}', shell=True, check=True, capture_output=True).stdout
slurm_array_job_id = int(output.split(b' ')[-1])
for task_id, chunk in enumerate(exp_array):
for exp in chunk:
if not unobserved:
collection.update_one(
{'_id': exp['_id']},
{'$set': {
'status': States.PENDING[0],
'slurm.array_id': slurm_array_job_id,
'slurm.task_id': task_id,
'slurm.sbatch_options': sbatch_options,
'seml.output_file': f"{output_dir_path}/{name}_{slurm_array_job_id}_{task_id}.out"}})
logging.verbose(f"Started experiment with array job ID {slurm_array_job_id}, task ID {task_id}.")
os.remove(path)
def add_experiments(db_collection_name,
config_file,
force_duplicates,
no_hash=False,
no_sanity_check=False,
no_code_checkpoint=False):
"""
Add configurations from a config file into the database.
Parameters
----------
db_collection_name: the MongoDB collection name.
config_file: path to the YAML configuration.
force_duplicates: if True, disable duplicate detection.
no_hash: if True, disable hashing of the configurations for duplicate detection. This is much slower, so use only
if you have a good reason to.
no_sanity_check: if True, do not check the config for missing/unused arguments.
no_code_checkpoint: if True, do not upload the experiment source code files to the MongoDB.
Returns
-------
None
"""
seml_config, slurm_config, experiment_config = read_config(config_file)
# Use current Anaconda environment if not specified
if 'conda_environment' not in seml_config:
if 'CONDA_DEFAULT_ENV' in os.environ:
seml_config['conda_environment'] = os.environ['CONDA_DEFAULT_ENV']
else:
seml_config['conda_environment'] = None
# Set Slurm config with default parameters as fall-back option
if slurm_config is None:
slurm_config = {'sbatch_options': {}}
for k, v in SETTINGS.SLURM_DEFAULT['sbatch_options'].items():
if k not in slurm_config['sbatch_options']:
slurm_config['sbatch_options'][k] = v
del SETTINGS.SLURM_DEFAULT['sbatch_options']
for k, v in SETTINGS.SLURM_DEFAULT.items():
if k not in slurm_config:
slurm_config[k] = v
# Check for and use sbatch options template
sbatch_options_template = slurm_config.get('sbatch_options_template', None)
if sbatch_options_template is not None:
if sbatch_options_template not in SETTINGS.SBATCH_OPTIONS_TEMPLATES:
raise ConfigError(
f"sbatch options template '{sbatch_options_template}' not found in settings.py."
)
for k, v in SETTINGS.SBATCH_OPTIONS_TEMPLATES[
sbatch_options_template].items():
if k not in slurm_config['sbatch_options']:
slurm_config['sbatch_options'][k] = v
del slurm_config['sbatch_options_template']
slurm_config['sbatch_options'] = remove_prepended_dashes(
slurm_config['sbatch_options'])
configs = generate_configs(experiment_config)
collection = get_collection(db_collection_name)
batch_id = get_max_in_collection(collection, "batch_id")
if batch_id is None:
batch_id = 1
else:
batch_id = batch_id + 1
if seml_config['use_uploaded_sources'] and not no_code_checkpoint:
uploaded_files = upload_sources(seml_config, collection, batch_id)
else:
uploaded_files = None
if not no_sanity_check:
check_config(seml_config['executable'],
seml_config['conda_environment'], configs)
path, commit, dirty = get_git_info(seml_config['executable'])
git_info = None
if path is not None:
git_info = {'path': path, 'commit': commit, 'dirty': dirty}
use_hash = not no_hash
if use_hash:
configs = [{**c, **{'config_hash': make_hash(c)}} for c in configs]
if not force_duplicates:
len_before = len(configs)
# First, check for duplicates withing the experiment configurations from the file.
if not use_hash:
# slow duplicate detection without hashes
unique_configs = []
for c in configs:
if c not in unique_configs:
unique_configs.append(c)
configs = unique_configs
else:
# fast duplicate detection using hashing.
configs_dict = {c['config_hash']: c for c in configs}
configs = [v for k, v in configs_dict.items()]
len_after_deduplication = len(configs)
# Now, check for duplicate configurations in the database.
configs = filter_experiments(collection, configs)
len_after = len(configs)
if len_after_deduplication != len_before:
logging.info(
f"{len_before - len_after_deduplication} of {len_before} experiment{s_if(len_before)} were "
f"duplicates. Adding only the {len_after_deduplication} unique configurations."
)
if len_after != len_after_deduplication:
logging.info(
f"{len_after_deduplication - len_after} of {len_after_deduplication} "
f"experiment{s_if(len_before)} were already found in the database. They were not added again."
)
# Create an index on the config hash. If the index is already present, this simply does nothing.
collection.create_index("config_hash")
# Add the configurations to the database with STAGED status.
if len(configs) > 0:
add_configs(collection, seml_config, slurm_config, configs,
uploaded_files, git_info)
def sample_parameter(parameter, samples, seed=None, parent_key=''):
"""
Generate random samples from the specified parameter.
The parameter types are inspired from https://github.com/hyperopt/hyperopt/wiki/FMin. When implementing new types,
please make them compatible with the hyperopt nomenclature so that we can switch to hyperopt at some point.
Parameters
----------
parameter: dict
Defines the type of parameter. Dict must include the key "type" that defines how the parameter will be sampled.
Supported types are
- choice: Randomly samples <samples> entries (with replacement) from the list in parameter['options']
- uniform: Uniformly samples between 'min' and 'max' as specified in the parameter dict.
- loguniform: Uniformly samples in log space between 'min' and 'max' as specified in the parameter dict.
- randint: Randomly samples integers between 'min' (included) and 'max' (excluded).
samples: int
Number of samples to draw for the parameter.
seed: int
The seed to use when drawing the parameter value. Defaults to None.
parent_key: str
The key to prepend the parameter name with. Used for nested parameters, where we here create a flattened version
where e.g. {'a': {'b': 11}, 'c': 3} becomes {'a.b': 11, 'c': 3}
Returns
-------
return_items: tuple(str, np.array or list)
tuple of the parameter name and a 1-D list/array of the samples drawn for the parameter.
"""
if "type" not in parameter:
raise ConfigError(f"No type found in parameter {parameter}")
return_items = []
allowed_keys = ['seed', 'type']
if seed is not None:
np.random.seed(seed)
elif 'seed' in parameter:
np.random.seed(parameter['seed'])
param_type = parameter['type']
if param_type == "choice":
choices = parameter['options']
allowed_keys.append("options")
sampled_values = [random.choice(choices) for _ in range(samples)]
return_items.append((parent_key, sampled_values))
elif param_type == "uniform":
min_val = parameter['min']
max_val = parameter['max']
allowed_keys.extend(['min', 'max'])
sampled_values = np.random.uniform(min_val, max_val, samples)
return_items.append((parent_key, sampled_values))
elif param_type == "loguniform":
if parameter['min'] <= 0:
raise ConfigError("Cannot take log of values <= 0")
min_val = np.log(parameter['min'])
max_val = np.log(parameter['max'])
allowed_keys.extend(['min', 'max'])
sampled_values = np.exp(np.random.uniform(min_val, max_val, samples))
return_items.append((parent_key, sampled_values))
elif param_type == "randint":
min_val = int(parameter['min'])
max_val = int(parameter['max'])
allowed_keys.extend(['min', 'max'])
sampled_values = np.random.randint(min_val, max_val, samples)
return_items.append((parent_key, sampled_values))
elif param_type == "randint_unique":
min_val = int(parameter['min'])
max_val = int(parameter['max'])
allowed_keys.extend(['min', 'max'])
sampled_values = np.random.choice(np.arange(min_val, max_val),
samples,
replace=False)
return_items.append((parent_key, sampled_values))
elif param_type == "parameter_collection":
sub_items = [
sample_parameter(v,
parent_key=f'{parent_key}.{k}',
seed=seed,
samples=samples)
for k, v in parameter['params'].items()
]
return_items.extend(
[sub_item for item in sub_items for sub_item in item])
else:
raise ConfigError(f"Parameter type {param_type} not implemented.")
if param_type != "parameter_collection":
extra_keys = set(parameter.keys()).difference(set(allowed_keys))
if len(extra_keys) > 0:
raise ConfigError(
f"Unexpected keys in parameter definition. Allowed keys for type '{param_type}' are "
f"{allowed_keys}. Unexpected keys: {extra_keys}")
return return_items
def generate_grid(parameter, parent_key=''):
"""
Generate a grid of parameter values from the input configuration.
Parameters
----------
parameter: dict
Defines the type of parameter. Options for parameter['type'] are
- choice: Expects a list of options in paramter['options'], which will be returned.
- range: Expects 'min', 'max', and 'step' keys with values in the dict that are used as
np.arange(min, max, step)
- uniform: Generates the grid using np.linspace(min, max, num, endpoint=True)
- loguniform: Uniformly samples 'num' points in log space (base 10) between 'min' and 'max'
- parameter_collection: wrapper around a dictionary of parameters (of the types above); we call this
function recursively on each of the sub-parameters.
parent_key: str
The key to prepend the parameter name with. Used for nested parameters, where we here create a flattened version
where e.g. {'a': {'b': 11}, 'c': 3} becomes {'a.b': 11, 'c': 3}
Returns
-------
return_items: tuple(str, list)
Name of the parameter and list containing the grid values for this parameter.
"""
if "type" not in parameter:
raise ConfigError(f"No type found in parameter {parameter}")
param_type = parameter['type']
allowed_keys = ['type']
return_items = []
if param_type == "choice":
values = parameter['options']
allowed_keys.append('options')
return_items.append((parent_key, values))
elif param_type == "range":
min_val = parameter['min']
max_val = parameter['max']
step = int(parameter['step'])
allowed_keys.extend(['min', 'max', 'step'])
values = list(np.arange(min_val, max_val, step))
return_items.append((parent_key, values))
elif param_type == "uniform":
min_val = parameter['min']
max_val = parameter['max']
num = int(parameter['num'])
allowed_keys.extend(['min', 'max', 'num'])
values = list(np.linspace(min_val, max_val, num, endpoint=True))
return_items.append((parent_key, values))
elif param_type == "loguniform":
min_val = parameter['min']
max_val = parameter['max']
num = int(parameter['num'])
allowed_keys.extend(['min', 'max', 'num'])
values = np.logspace(np.log10(min_val),
np.log10(max_val),
num,
endpoint=True)
return_items.append((parent_key, values))
elif param_type == "parameter_collection":
sub_items = [
generate_grid(v, parent_key=f'{parent_key}.{k}')
for k, v in parameter['params'].items()
]
return_items.extend(
[sub_item for item in sub_items for sub_item in item])
else:
raise ConfigError(f"Parameter {param_type} not implemented.")
if param_type != "parameter_collection":
extra_keys = set(parameter.keys()).difference(set(allowed_keys))
if len(extra_keys) > 0:
raise ConfigError(
f"Unexpected keys in parameter definition. Allowed keys for type '{param_type}' are "
f"{allowed_keys}. Unexpected keys: {extra_keys}")
return return_items
def generate_configs(experiment_config):
"""Generate parameter configurations based on an input configuration.
Input is a nested configuration where on each level there can be 'fixed', 'grid', and 'random' parameters.
In essence, we take the cartesian product of all the `grid` parameters and take random samples for the random
parameters. The nested structure makes it possible to define different parameter spaces e.g. for different datasets.
Parameter definitions lower in the hierarchy overwrite parameters defined closer to the root.
For each leaf configuration we take the maximum of all num_samples values on the path since we need to have the same
number of samples for each random parameter.
For each configuration of the `grid` parameters we then create `num_samples` configurations of the random
parameters, i.e. leading to `num_samples * len(grid_configurations)` configurations.
See Also `examples/example_config.yaml` and the example below.
Parameters
----------
experiment_config: dict
Dictionary that specifies the "search space" of parameters that will be enumerated. Should be
parsed from a YAML file.
Returns
-------
all_configs: list of dicts
Contains the individual combinations of the parameters.
"""
reserved, next_level = unpack_config(experiment_config)
reserved = standardize_config(reserved)
if not any([len(reserved.get(k, {})) > 0 for k in RESERVED_KEYS]):
raise ConfigError(
"No parameters defined under grid, fixed, or random in the config file."
)
level_stack = [('', next_level)]
config_levels = [reserved]
final_configs = []
detect_duplicate_parameters(invert_config(reserved), None)
while len(level_stack) > 0:
current_sub_name, sub_vals = level_stack.pop(0)
sub_config, sub_levels = unpack_config(sub_vals)
if current_sub_name != '' and not any(
[len(sub_config.get(k, {})) > 0 for k in RESERVED_KEYS]):
raise ConfigError(
f"No parameters defined under grid, fixed, or random in sub-config {current_sub_name}."
)
sub_config = standardize_config(sub_config)
config_above = config_levels.pop(0)
inverted_sub_config = invert_config(sub_config)
detect_duplicate_parameters(inverted_sub_config, current_sub_name)
inverted_config_above = invert_config(config_above)
redefined_parameters = set(inverted_sub_config.keys()).intersection(
set(inverted_config_above.keys()))
if len(redefined_parameters) > 0:
logging.info(
f"Found redefined parameters in sub-config '{current_sub_name}': {redefined_parameters}. "
f"Definitions in sub-configs override more general ones.")
config_above = copy.deepcopy(config_above)
for p in redefined_parameters:
sections = inverted_config_above[p]
for s in sections:
del config_above[s][p]
config = merge_dicts(config_above, sub_config)
if len(sub_levels) == 0:
final_configs.append((current_sub_name, config))
for sub_name, sub_vals in sub_levels.items():
new_sub_name = f'{current_sub_name}.{sub_name}' if current_sub_name != '' else sub_name
level_stack.append((new_sub_name, sub_vals))
config_levels.append(config)
all_configs = []
for subconfig_name, conf in final_configs:
conf = standardize_config(conf)
random_params = conf['random'] if 'random' in conf else {}
fixed_params = flatten(conf['fixed']) if 'fixed' in conf else {}
grid_params = conf['grid'] if 'grid' in conf else {}
if len(random_params) > 0:
num_samples = random_params['samples']
root_seed = random_params.get('seed', None)
random_sampled = sample_random_configs(flatten(random_params),
seed=root_seed,
samples=num_samples)
grids = [
generate_grid(v, parent_key=k) for k, v in grid_params.items()
]
grid_configs = dict([sub for item in grids for sub in item])
grid_product = list(cartesian_product_dict(grid_configs))
with_fixed = [{**d, **fixed_params} for d in grid_product]
if len(random_params) > 0:
with_random = [{
**grid,
**random
} for grid in with_fixed for random in random_sampled]
else:
with_random = with_fixed
all_configs.extend(with_random)
# Cast NumPy integers to normal integers since PyMongo doesn't like them
all_configs = [{
k: int(v) if isinstance(v, np.integer) else v
for k, v in config.items()
} for config in all_configs]
all_configs = [unflatten(conf) for conf in all_configs]
return all_configs