def cross_validation_stat(cfg: IGenericTaskConfig,
metric,
stage=None,
threshold=0.5,
folds=None):
metrics = []
cfg.get_dataset() # this is actually needed
mDict = {}
if not folds:
folds = list(range(cfg.folds_count))
if hasattr(cfg, "_folds"):
folds = cfg._folds()
if isFinal(metric):
fnc = configloader.load("layers").catalog[metric]
if hasattr(fnc, "func"):
fnc = fnc.func
for i in folds:
fa = FoldsAndStages(cfg, i, stage)
val = cfg.predictions("validation", i, stage)
pv = fnc(fa, val)
if isinstance(pv, dict):
for k in pv:
if k not in mDict:
mDict[k] = []
mDict[k].append(pv[k])
else:
metrics.append(pv)
if len(mDict) > 0:
rs = {}
for c in mDict:
rs[c] = stat(mDict[c])
return rs
return stat(metrics)
for i in folds:
if cfg._reporter is not None and cfg._reporter.isCanceled():
return {"canceled": True}
predictionDS = get_validation_prediction(cfg, i, stage)
val = considerThreshold(predictionDS, metric, threshold)
eval_metric = generic_config.eval_metric(val, metric,
cfg.get_eval_batch())
metrics.append(np.mean(eval_metric))
return stat(metrics)
def holdout_stat(cfg:IGenericTaskConfig, metric,stage=None,threshold=0.5):
if cfg._reporter is not None and cfg._reporter.isCanceled():
return {"canceled": True}
if isFinal(metric):
fnc=configloader.load("layers").catalog[metric]
if hasattr(fnc, "func"):
fnc=fnc.func
for i in range(cfg.folds_count):
fa=FoldsAndStages(cfg,i,stage)
val=cfg.predictions("holdout",i,stage)
r = fnc(fa, val)
if isinstance(r, dict):
return r
return float(r)
predictionDS = get_holdout_prediction(cfg, None, stage)
val = considerThreshold(predictionDS, metric, threshold)
eval_metric = generic_config.eval_metric(val, metric, cfg.get_eval_batch())
if cfg._reporter is not None and cfg._reporter.isCanceled():
return {"canceled": True}
return float(np.mean(eval_metric))