def recommender(
experiment: str,
config_path: str,
data_path: str,
evaluations_path: str,
nskip: int,
):
"""
Evaluates the performance of the models proposed by a recommender on a hold-out dataset.
Recommendations are made for each dataset by providing recommenders with the offline
evaluations on the remaining datasets in the registry.
This call runs the Sacred script for each provided configuration sequentially and returns only
once all runs have completed.
"""
with Path(config_path).open("r", encoding="utf-8") as f:
content = yaml.safe_load(f)
configs = explode_key_values("recommender", content)
for configuration in iterate_configurations(configs, nskip):
run_sacred_script(
"recommender.py",
experiment=experiment,
data_path=data_path,
evaluations_path=evaluations_path,
**configuration,
)
def ensemble(
experiment: str,
config_path: str,
data_path: str,
evaluations_path: str,
nskip: int,
):
"""
Evaluates the performance of an ensemble of best-performing models. This
allows also to only consider models of a particular type (thus building
hyper-ensembles).
This call runs the Sacred script for each provided configuration
sequentially and returns only once all runs have completed.
"""
with Path(config_path).open("r", encoding="utf-8") as f:
content = yaml.safe_load(f)
configs = explode_key_values("__", content)
for configuration in iterate_configurations(configs, nskip):
run_sacred_script(
"ensemble.py",
experiment=experiment,
data_path=data_path,
evaluations_path=evaluations_path,
**{k: v
for k, v in configuration.items() if k != "__"},
)
def surrogate(
experiment: str,
config_path: str,
data_path: str,
evaluations_path: str,
nskip: int,
):
"""
Evaluates the performance of a set of surrogate models using the available
offline evaluations. Performance is evaluated via ranking metrics and
performed via stratified leave-one-out cross-validation where each stratum
consists of the evaluations on a single evaluation dataset.
This call runs the Sacred script for each provided configuration
sequentially and returns only once all runs have completed.
"""
with Path(config_path).open("r", encoding="utf-8") as f:
content = yaml.safe_load(f)
configs = explode_key_values("surrogate", content)
for configuration in iterate_configurations(configs, nskip):
run_sacred_script(
"surrogate.py",
experiment=experiment,
data_path=data_path,
evaluations_path=evaluations_path,
**configuration,
)