def _get_mean_costs(self, incs: typing.List[Configuration], new_rh: RunHistory): """ Compute mean cost per instance Parameters ---------- incs : typing.List[Configuration] incumbents determined by all parallel SMAC runs new_rh : RunHistory runhistory to determine mean performance Returns ------- List[float] means Dict(Config -> Dict(inst_id(str) -> float)) """ config_cost_per_inst = {} results = [] for incumbent in incs: cost_per_inst = new_rh.get_instance_costs_for_config( config=incumbent) config_cost_per_inst[incumbent] = cost_per_inst values = list(cost_per_inst.values()) if values: results.append(np.mean(values)) else: results.append(np.nan) return results, config_cost_per_inst
def get_cost_dict_for_config(rh: RunHistory, conf: Configuration, par: int = 1, cutoff: typing.Union[float, None] = None): """ Aggregates loss for configuration on evaluated instances over seeds. Parameters ---------- rh: RunHistory runhistory with data conf: Configuration configuration to evaluate par: int par-factor with which to multiply timeouts cutoff: float cutoff of scenario - used to penalize costs if par != 1 Returns ------- cost: dict(instance->cost) cost per instance (aggregated or as list per seed) """ instance_costs = rh.get_instance_costs_for_config(conf) if par != 1: if cutoff: instance_costs = { k: v if v < cutoff else v * par for k, v in instance_costs.items() } else: raise ValueError( "To apply penalization of costs, a cutoff needs to be provided." ) return instance_costs