def recommender( experiment: str, config_path: str, data_path: str, evaluations_path: str, nskip: int, ): """ Evaluates the performance of the models proposed by a recommender on a hold-out dataset. Recommendations are made for each dataset by providing recommenders with the offline evaluations on the remaining datasets in the registry. This call runs the Sacred script for each provided configuration sequentially and returns only once all runs have completed. """ with Path(config_path).open("r", encoding="utf-8") as f: content = yaml.safe_load(f) configs = explode_key_values("recommender", content) for configuration in iterate_configurations(configs, nskip): run_sacred_script( "recommender.py", experiment=experiment, data_path=data_path, evaluations_path=evaluations_path, **configuration, )
def ensemble( experiment: str, config_path: str, data_path: str, evaluations_path: str, nskip: int, ): """ Evaluates the performance of an ensemble of best-performing models. This allows also to only consider models of a particular type (thus building hyper-ensembles). This call runs the Sacred script for each provided configuration sequentially and returns only once all runs have completed. """ with Path(config_path).open("r", encoding="utf-8") as f: content = yaml.safe_load(f) configs = explode_key_values("__", content) for configuration in iterate_configurations(configs, nskip): run_sacred_script( "ensemble.py", experiment=experiment, data_path=data_path, evaluations_path=evaluations_path, **{k: v for k, v in configuration.items() if k != "__"}, )
def surrogate( experiment: str, config_path: str, data_path: str, evaluations_path: str, nskip: int, ): """ Evaluates the performance of a set of surrogate models using the available offline evaluations. Performance is evaluated via ranking metrics and performed via stratified leave-one-out cross-validation where each stratum consists of the evaluations on a single evaluation dataset. This call runs the Sacred script for each provided configuration sequentially and returns only once all runs have completed. """ with Path(config_path).open("r", encoding="utf-8") as f: content = yaml.safe_load(f) configs = explode_key_values("surrogate", content) for configuration in iterate_configurations(configs, nskip): run_sacred_script( "surrogate.py", experiment=experiment, data_path=data_path, evaluations_path=evaluations_path, **configuration, )