def cross_validation_stat(cfg: IGenericTaskConfig, metric, stage=None, threshold=0.5, folds=None): metrics = [] cfg.get_dataset() # this is actually needed mDict = {} if not folds: folds = list(range(cfg.folds_count)) if hasattr(cfg, "_folds"): folds = cfg._folds() if isFinal(metric): fnc = configloader.load("layers").catalog[metric] if hasattr(fnc, "func"): fnc = fnc.func for i in folds: fa = FoldsAndStages(cfg, i, stage) val = cfg.predictions("validation", i, stage) pv = fnc(fa, val) if isinstance(pv, dict): for k in pv: if k not in mDict: mDict[k] = [] mDict[k].append(pv[k]) else: metrics.append(pv) if len(mDict) > 0: rs = {} for c in mDict: rs[c] = stat(mDict[c]) return rs return stat(metrics) for i in folds: if cfg._reporter is not None and cfg._reporter.isCanceled(): return {"canceled": True} predictionDS = get_validation_prediction(cfg, i, stage) val = considerThreshold(predictionDS, metric, threshold) eval_metric = generic_config.eval_metric(val, metric, cfg.get_eval_batch()) metrics.append(np.mean(eval_metric)) return stat(metrics)
def holdout_stat(cfg:IGenericTaskConfig, metric,stage=None,threshold=0.5): if cfg._reporter is not None and cfg._reporter.isCanceled(): return {"canceled": True} if isFinal(metric): fnc=configloader.load("layers").catalog[metric] if hasattr(fnc, "func"): fnc=fnc.func for i in range(cfg.folds_count): fa=FoldsAndStages(cfg,i,stage) val=cfg.predictions("holdout",i,stage) r = fnc(fa, val) if isinstance(r, dict): return r return float(r) predictionDS = get_holdout_prediction(cfg, None, stage) val = considerThreshold(predictionDS, metric, threshold) eval_metric = generic_config.eval_metric(val, metric, cfg.get_eval_batch()) if cfg._reporter is not None and cfg._reporter.isCanceled(): return {"canceled": True} return float(np.mean(eval_metric))