def createNet(self):
if self.architecture in configloader.load("layers").catalog:
clazz = configloader.load("layers").catalog[self.architecture].func
elif self.architecture.lower() in backbones.get_names():
clazz = create_back_bone(self.architecture)
else:
try:
clazz = classification_models.Classifiers.get_classifier(
self.architecture.lower())
except:
try:
clazz = getattr(apps, self.architecture)
except:
print("Unknown architeture:" + self.backbone)
print("Known archectures:",
classification_models.Classifiers.names())
raise ValueError("Unknown architecture")
t: configloader.Type = configloader.loaded['classification'].catalog[
'ClassificationPipeline']
r = t.customProperties()
cleaned = {}
for arg in self.all:
pynama = t.alias(arg)
if not arg in r:
cleaned[pynama] = self.all[arg]
self.clean(cleaned)
if self.crops is not None:
cleaned["input_shape"] = (cleaned["input_shape"][0] // self.crops,
cleaned["input_shape"][1] // self.crops,
cleaned["input_shape"][2])
cleaned["input_shape"] = tuple(cleaned["input_shape"])
cleaned["include_top"] = False
model1 = self.__inner_create(clazz, cleaned)
cuout = model1.output
if len(cuout.shape) == 4:
cuout = keras.layers.GlobalAveragePooling2D()(cuout)
ac = self.all["activation"]
if ac == "none":
ac = None
if self.dropout > 0:
cuout = keras.layers.Dropout(self.dropout)(cuout)
if isinstance(ac, list):
ls = []
for i in range(len(ac)):
dl = keras.layers.Dense(self.all["classes"][i],
activation=ac[i])(cuout)
ls.append(dl)
model = keras.Model(model1.input, ls)
else:
dl = keras.layers.Dense(self.all["classes"], activation=ac)(cuout)
model = keras.Model(model1.input, dl)
return model
def cross_validation_stat(cfg: IGenericTaskConfig,
metric,
stage=None,
threshold=0.5,
folds=None):
metrics = []
cfg.get_dataset() # this is actually needed
mDict = {}
if not folds:
folds = list(range(cfg.folds_count))
if hasattr(cfg, "_folds"):
folds = cfg._folds()
if isFinal(metric):
fnc = configloader.load("layers").catalog[metric]
if hasattr(fnc, "func"):
fnc = fnc.func
for i in folds:
fa = FoldsAndStages(cfg, i, stage)
val = cfg.predictions("validation", i, stage)
pv = fnc(fa, val)
if isinstance(pv, dict):
for k in pv:
if k not in mDict:
mDict[k] = []
mDict[k].append(pv[k])
else:
metrics.append(pv)
if len(mDict) > 0:
rs = {}
for c in mDict:
rs[c] = stat(mDict[c])
return rs
return stat(metrics)
for i in folds:
if cfg._reporter is not None and cfg._reporter.isCanceled():
return {"canceled": True}
predictionDS = get_validation_prediction(cfg, i, stage)
val = considerThreshold(predictionDS, metric, threshold)
eval_metric = generic_config.eval_metric(val, metric,
cfg.get_eval_batch())
metrics.append(np.mean(eval_metric))
return stat(metrics)
def holdout_stat(cfg:IGenericTaskConfig, metric,stage=None,threshold=0.5):
if cfg._reporter is not None and cfg._reporter.isCanceled():
return {"canceled": True}
if isFinal(metric):
fnc=configloader.load("layers").catalog[metric]
if hasattr(fnc, "func"):
fnc=fnc.func
for i in range(cfg.folds_count):
fa=FoldsAndStages(cfg,i,stage)
val=cfg.predictions("holdout",i,stage)
r = fnc(fa, val)
if isinstance(r, dict):
return r
return float(r)
predictionDS = get_holdout_prediction(cfg, None, stage)
val = considerThreshold(predictionDS, metric, threshold)
eval_metric = generic_config.eval_metric(val, metric, cfg.get_eval_batch())
if cfg._reporter is not None and cfg._reporter.isCanceled():
return {"canceled": True}
return float(np.mean(eval_metric))
def createNet(self):
ac = self.all["activation"]
if ac == "none":
ac = None
self.all["activation"] = ac
if self.architecture in custom_models:
clazz = custom_models[self.architecture]
else:
if self.architecture in configloader.load("layers").catalog:
clazz = configloader.load("layers").catalog[
self.architecture].func
else:
if not self.architecture in dir(segmentation_models):
print("Unknown architecture:" + self.backbone)
print("Known architectures:",
["FPN", "Unet", "Linknet", "PSPNet", "DeeplabV3"])
raise ValueError("Unknown architecture")
clazz = getattr(segmentation_models, self.architecture)
if not self.backbone.lower(
) in classification_models.Classifiers.names():
print("Unknown backbone:" + self.backbone)
print("Known backbones:",
classification_models.Classifiers.names())
raise ValueError("Unknown backbone")
t: configloader.Type = configloader.loaded['segmentation'].catalog[
'PipelineConfig']
r = t.customProperties()
self.backbone = self.backbone.lower()
cleaned = {}
sig = inspect.signature(clazz)
for arg in self.all:
pynama = t.alias(arg)
if not arg in r and pynama in sig.parameters:
cleaned[pynama] = self.all[arg]
self.clean(cleaned)
if self.crops is not None:
cleaned["input_shape"] = (cleaned["input_shape"][0] // self.crops,
cleaned["input_shape"][1] // self.crops,
cleaned["input_shape"][2])
if "input_shape" in cleaned:
if cleaned["input_shape"][
2] > 3 and self.encoder_weights != None and len(
self.encoder_weights) > 0:
if os.path.exists(self.path + ".mdl-nchannel"):
cleaned["encoder_weights"] = None
model = clazz(**cleaned)
model.load_weights(self.path + ".mdl-nchannel")
return model
copy = cleaned.copy()
copy["input_shape"] = (cleaned["input_shape"][0],
cleaned["input_shape"][1], 3)
model1 = clazz(**copy)
cleaned["encoder_weights"] = None
model = clazz(**cleaned)
self.adaptNet(model, model1, self.copyWeights)
model.save_weights(self.path + ".mdl-nchannel")
return model
return clazz(**cleaned)
def augmenter(aug_class):
configloader.load("augmenters").register_member(aug_class.__name__, aug_class)
if not issubclass(aug_class, Augmenter):
raise ValueError('{} is not an instance of imgaug.augmenters.Augmenter!'.format(aug_class.__name__))
return aug_class
from musket_core.utils import load_yaml, save, load
import keras
from keras import backend as K
import musket_core.templating as tp
from musket_core.caches import *
from musket_core import datasets
from musket_core import crf
from builtins import isinstance
from keras.layers.core import Lambda
import collections
from musket_core.preprocessing import SplitPreproccessor, SplitConcatPreprocessor, Augmentation, take_nth
import musket_core.builtin_datasets
import musket_core.builtin_trainables
import importlib
layers = configloader.load("layers")
layers.register_module(crf)
def take_input(layers, declarations, config, outputs, linputs, pName,
withArgs):
def a(args):
return args
return a
def seq(layers, declarations, config, outputs, linputs, pName, withArgs):
layers = Layers(config, declarations, {}, outputs, linputs, withArgs)
return layers
def final_metric(func):
func.final_metric = True
configloader.load("layers").catalog[func.__name__] = func
return func
return float(thresholded) # Or thresholded.mean()
metricNames = ["dice", "iou", "map10"]
ms = {"dice": Dice, "iou": IOU, "map10": MAP10}
treshold = [True, False]
zero_is = [0, 1, None]
for m in metricNames:
for t in treshold:
for z in zero_is:
mName = m
if t:
mName = mName + "_with_custom_treshold_"
else:
mName = mName + "_"
if z == 0:
mName = mName + "true_negative_is_zero"
elif z == 1:
mName = mName + "true_negative_is_one"
else:
mName = mName + "true_negative_is_skip"
fnc = ms[m](z)
fnc.__name__ = mName
if t:
fnc = WithTreshold(fnc)
else:
fnc = SimpleMetric(fnc)
configloader.load("layers").catalog[fnc.name] = fnc