def __init__(self,
log,
mainOutputAbsFolder,
folderForPredictions,
folderForFeatures,
num_classes,
cfg):
#Importants for running session.
# From Session:
self.log = log
self.mainOutputAbsFolder = mainOutputAbsFolder
# From test config:
self.sessionName = self.getSessionName( cfg[cfg.SESSION_NAME] )
abs_path_to_cfg = cfg.get_abs_path_to_cfg()
abs_path_to_saved = getAbsPathEvenIfRelativeIsGiven( cfg[cfg.SAVED_MODEL], abs_path_to_cfg ) if cfg[cfg.SAVED_MODEL] is not None else None # Where to load the model from.
self.savedModelFilepath = check_and_adjust_path_to_ckpt( self.log, abs_path_to_saved) if abs_path_to_saved is not None else None
#Input:
#[[case1-ch1, ..., caseN-ch1], [case1-ch2,...,caseN-ch2]]
listOfAListPerChannelWithFilepathsOfAllCases = [parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(channelConfPath, abs_path_to_cfg)) for channelConfPath in cfg[cfg.CHANNELS]]
self.channelsFilepaths = [ list(item) for item in zip(*tuple(listOfAListPerChannelWithFilepathsOfAllCases)) ] # [[case1-ch1, case1-ch2], ..., [caseN-ch1, caseN-ch2]]
self.providedGt = True if cfg[cfg.GT_LABELS] is not None else False
self.gtLabelsFilepaths = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS], abs_path_to_cfg) ) if cfg[cfg.GT_LABELS] is not None else None
self.providedRoiMasks = True if cfg[cfg.ROI_MASKS] is not None else False
self.roiMasksFilepaths = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.ROI_MASKS], abs_path_to_cfg) ) if self.providedRoiMasks else None
#Output:
self.namesToSavePredictionsAndFeatures = parseFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.NAMES_FOR_PRED_PER_CASE], abs_path_to_cfg) ) if cfg[cfg.NAMES_FOR_PRED_PER_CASE] is not None else None #CAREFUL: different parser! #Optional. Not required if not saving results.
#predictions
self.saveSegmentation = cfg[cfg.SAVE_SEGM] if cfg[cfg.SAVE_SEGM] is not None else True
self.saveProbMapsBoolPerClass = cfg[cfg.SAVE_PROBMAPS_PER_CLASS] if (cfg[cfg.SAVE_PROBMAPS_PER_CLASS] is not None and cfg[cfg.SAVE_PROBMAPS_PER_CLASS] != []) else [True]*num_classes
self.filepathsToSavePredictionsForEachPatient = None #Filled by call to self.makeFilepathsForPredictionsAndFeatures()
#features:
self.saveIndividualFmImages = cfg[cfg.SAVE_INDIV_FMS] if cfg[cfg.SAVE_INDIV_FMS] is not None else False
self.saveMultidimensionalImageWithAllFms = cfg[cfg.SAVE_4DIM_FMS] if cfg[cfg.SAVE_4DIM_FMS] is not None else False
if self.saveIndividualFmImages == True or self.saveMultidimensionalImageWithAllFms == True:
indices_fms_per_pathtype_per_layer_to_save = [cfg[cfg.INDICES_OF_FMS_TO_SAVE_NORMAL]] +\
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_SUBSAMPLED]] +\
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_FC]]
self.indices_fms_per_pathtype_per_layer_to_save = [item if item is not None else [] for item in indices_fms_per_pathtype_per_layer_to_save]
else:
self.indices_fms_per_pathtype_per_layer_to_save = None
self.filepathsToSaveFeaturesForEachPatient = None #Filled by call to self.makeFilepathsForPredictionsAndFeatures()
#Preprocessing
self.padInputImagesBool = cfg[cfg.PAD_INPUT] if cfg[cfg.PAD_INPUT] is not None else True
#Others useful internally or for reporting:
self.numberOfCases = len(self.channelsFilepaths)
#HIDDENS, no config allowed for these at the moment:
self.useSameSubChannelsAsSingleScale = True
self.subsampledChannelsFilepaths = "placeholder" #List of Lists with filepaths per patient. Only used when above is False.
self._makeFilepathsForPredictionsAndFeatures( folderForPredictions, folderForFeatures )
def __init__(self,
log,
mainOutputAbsFolder,
folderForPredictions,
folderForFeatures,
num_classes,
cfg):
#Importants for running session.
# From Session:
self.log = log
self.mainOutputAbsFolder = mainOutputAbsFolder
# From test config:
self.sessionName = self.getSessionName( cfg[cfg.SESSION_NAME] )
abs_path_to_cfg = cfg.get_abs_path_to_cfg()
abs_path_to_saved = getAbsPathEvenIfRelativeIsGiven( cfg[cfg.SAVED_MODEL], abs_path_to_cfg ) if cfg[cfg.SAVED_MODEL] is not None else None # Where to load the model from.
self.savedModelFilepath = check_and_adjust_path_to_ckpt( self.log, abs_path_to_saved) if abs_path_to_saved is not None else None
#Input:
#[[case1-ch1, ..., caseN-ch1], [case1-ch2,...,caseN-ch2]]
listOfAListPerChannelWithFilepathsOfAllCases = [parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(channelConfPath, abs_path_to_cfg)) for channelConfPath in cfg[cfg.CHANNELS]]
self.channelsFilepaths = [ list(item) for item in zip(*tuple(listOfAListPerChannelWithFilepathsOfAllCases)) ] # [[case1-ch1, case1-ch2], ..., [caseN-ch1, caseN-ch2]]
self.providedGt = True if cfg[cfg.GT_LABELS] is not None else False
self.gtLabelsFilepaths = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS], abs_path_to_cfg) ) if cfg[cfg.GT_LABELS] is not None else None
self.providedRoiMasks = True if cfg[cfg.ROI_MASKS] is not None else False
self.roiMasksFilepaths = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.ROI_MASKS], abs_path_to_cfg) ) if self.providedRoiMasks else None
#Output:
self.namesToSavePredictionsAndFeatures = parseFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.NAMES_FOR_PRED_PER_CASE], abs_path_to_cfg) ) if cfg[cfg.NAMES_FOR_PRED_PER_CASE] is not None else None #CAREFUL: different parser! #Optional. Not required if not saving results.
#predictions
self.saveSegmentation = cfg[cfg.SAVE_SEGM] if cfg[cfg.SAVE_SEGM] is not None else True
self.saveProbMapsBoolPerClass = cfg[cfg.SAVE_PROBMAPS_PER_CLASS] if (cfg[cfg.SAVE_PROBMAPS_PER_CLASS] is not None and cfg[cfg.SAVE_PROBMAPS_PER_CLASS] != []) else [True]*num_classes
self.filepathsToSavePredictionsForEachPatient = None #Filled by call to self.makeFilepathsForPredictionsAndFeatures()
self.suffixForSegmAndProbsDict = cfg[cfg.SUFFIX_SEGM_PROB] if cfg[cfg.SUFFIX_SEGM_PROB] is not None else {"segm": "Segm", "prob": "ProbMapClass"}
self.batchsize = cfg[cfg.BATCHSIZE] if cfg[cfg.BATCHSIZE] is not None else 10
#features:
self.saveIndividualFmImages = cfg[cfg.SAVE_INDIV_FMS] if cfg[cfg.SAVE_INDIV_FMS] is not None else False
self.saveMultidimensionalImageWithAllFms = cfg[cfg.SAVE_4DIM_FMS] if cfg[cfg.SAVE_4DIM_FMS] is not None else False
if self.saveIndividualFmImages == True or self.saveMultidimensionalImageWithAllFms == True:
indices_fms_per_pathtype_per_layer_to_save = [cfg[cfg.INDICES_OF_FMS_TO_SAVE_NORMAL]] +\
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_SUBSAMPLED]] +\
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_FC]]
self.indices_fms_per_pathtype_per_layer_to_save = [item if item is not None else [] for item in indices_fms_per_pathtype_per_layer_to_save]
else:
self.indices_fms_per_pathtype_per_layer_to_save = None
self.filepathsToSaveFeaturesForEachPatient = None #Filled by call to self.makeFilepathsForPredictionsAndFeatures()
#Preprocessing
self.padInputImagesBool = cfg[cfg.PAD_INPUT] if cfg[cfg.PAD_INPUT] is not None else True
#Others useful internally or for reporting:
self.numberOfCases = len(self.channelsFilepaths)
#HIDDENS, no config allowed for these at the moment:
self._makeFilepathsForPredictionsAndFeatures( folderForPredictions, folderForFeatures )
def make_output_folders(self):
self._main_out_folder_abs = getAbsPathEvenIfRelativeIsGiven( self._cfg[self._cfg.FOLDER_OUTP],
self.get_abs_path_to_cfg() )
[self._log_folder_abs,
self._out_folder_models,
self._out_folder_preds,
self._out_folder_fms] = makeFoldersNeededForTrainingSession( self._main_out_folder_abs, self._sess_name )
def __init__(self, cfg):
self._cfg = cfg
self._sess_name = self._make_sess_name()
self._main_out_folder_abs = getAbsPathEvenIfRelativeIsGiven( cfg[cfg.FOLDER_OUTP], self.get_abs_path_to_cfg() )
self._log_folder_abs = None
self._log = None
def make_output_folders(self):
self._main_out_folder_abs = getAbsPathEvenIfRelativeIsGiven( self._cfg[self._cfg.FOLDER_OUTP],
self.get_abs_path_to_cfg() )
[self._log_folder_abs,
self._out_folder_models,
self._out_folder_preds,
self._out_folder_fms] = makeFoldersNeededForTrainingSession( self._main_out_folder_abs, self._sess_name )
def __init__(self, log, mainOutputAbsFolder, folderForSessionCnnModels,
folderForPredictionsVal, folderForFeaturesVal, num_classes,
model_name, cfg):
# Importants for running session.
# From Session:
self.log = log
self.mainOutputAbsFolder = mainOutputAbsFolder
# From Config:
self.sessionName = self.getSessionName(cfg[cfg.SESSION_NAME])
abs_path_to_cfg = cfg.get_abs_path_to_cfg()
abs_path_to_saved = getAbsPathEvenIfRelativeIsGiven(cfg[cfg.SAVED_MODEL], abs_path_to_cfg) \
if cfg[cfg.SAVED_MODEL] is not None else None # Load pretrained model.
self.savedModelFilepath = check_and_adjust_path_to_ckpt(self.log, abs_path_to_saved) \
if abs_path_to_saved is not None else None
self.tensorboardLog = cfg[cfg.TENSORBOARD_LOG] if cfg[
cfg.TENSORBOARD_LOG] is not None else False
# ====================TRAINING==========================
self.filepath_to_save_models = folderForSessionCnnModels + "/" + model_name + "." + self.sessionName
if cfg[cfg.CHANNELS_TR] is None:
self.errReqChansTr()
if cfg[cfg.GT_LABELS_TR] is None:
self.errReqGtTr()
# [[case1-ch1, ..., caseN-ch1], [case1-ch2,...,caseN-ch2]]
listOfAListPerChannelWithFilepathsOfAllCasesTrain = [
parseAbsFileLinesInList(
getAbsPathEvenIfRelativeIsGiven(channelConfPath,
abs_path_to_cfg))
for channelConfPath in cfg[cfg.CHANNELS_TR]
]
# [[case1-ch1, case1-ch2], ..., [caseN-ch1, caseN-ch2]]
self.channelsFilepathsTrain = \
[list(item) for item in zip(*tuple(listOfAListPerChannelWithFilepathsOfAllCasesTrain))]
self.gtLabelsFilepathsTrain = \
parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS_TR], abs_path_to_cfg))
# [Optionals]
# ~~~~~~~~~Sampling~~~~~~~
self.roiMasksFilepathsTrain = \
parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(cfg[cfg.ROI_MASKS_TR], abs_path_to_cfg)) \
if cfg[cfg.ROI_MASKS_TR] is not None else None
sampling_type_flag_tr = cfg[cfg.TYPE_OF_SAMPLING_TR] if cfg[
cfg.TYPE_OF_SAMPLING_TR] is not None else 3
self.sampling_type_inst_tr = samplingType.SamplingType(
self.log, sampling_type_flag_tr, num_classes)
if sampling_type_flag_tr in [
0, 3
] and cfg[cfg.PROP_OF_SAMPLES_PER_CAT_TR] is not None:
self.sampling_type_inst_tr.set_perc_of_samples_per_cat(
cfg[cfg.PROP_OF_SAMPLES_PER_CAT_TR])
else:
n_sampl_cats_tr = self.sampling_type_inst_tr.get_n_sampling_cats()
self.sampling_type_inst_tr.set_perc_of_samples_per_cat(
[1.0 / n_sampl_cats_tr] * n_sampl_cats_tr)
self.paths_to_wmaps_per_sampl_cat_per_subj_train = None
if cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_TR] is not None:
# [[case1-weightMap1, ..., caseN-weightMap1], [case1-weightMap2,...,caseN-weightMap2]]
self.paths_to_wmaps_per_sampl_cat_per_subj_train = [
parseAbsFileLinesInList(
getAbsPathEvenIfRelativeIsGiven(weightMapConfPath,
abs_path_to_cfg)) for
weightMapConfPath in cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_TR]
]
# ~~~~~~~~ Training Cycle ~~~~~~~~~~~
self.numberOfEpochs = cfg[cfg.NUM_EPOCHS] if cfg[
cfg.NUM_EPOCHS] is not None else 35
self.numberOfSubepochs = cfg[cfg.NUM_SUBEP] if cfg[
cfg.NUM_SUBEP] is not None else 20
self.max_n_cases_per_subep_train = \
cfg[cfg.NUM_CASES_LOADED_PERSUB] if cfg[cfg.NUM_CASES_LOADED_PERSUB] is not None else 50
self.n_samples_per_subep_train = \
cfg[cfg.NUM_TR_SEGMS_LOADED_PERSUB] if cfg[cfg.NUM_TR_SEGMS_LOADED_PERSUB] is not None else 1000
self.batchsize_train = cfg[cfg.BATCHSIZE_TR] if cfg[
cfg.BATCHSIZE_TR] is not None else errReqBatchSizeTr()
self.num_parallel_proc_sampling = cfg[cfg.NUM_OF_PROC_SAMPL] if cfg[
cfg.NUM_OF_PROC_SAMPL] is not None else 0
# ~~~~~~~ Learning Rate Schedule ~~~~~~~~
assert cfg[cfg.LR_SCH_TYPE] in [
'stable', 'predef', 'poly', 'auto', 'expon'
]
self.lr_sched_params = {
'type': cfg[cfg.LR_SCH_TYPE]
if cfg[cfg.LR_SCH_TYPE] is not None else 'poly',
'predef': {
'epochs':
cfg[cfg.PREDEF_SCH],
'div_lr_by':
cfg[cfg.DIV_LR_BY] if cfg[cfg.DIV_LR_BY] is not None else 2.0
},
'auto': {
'min_incr_of_val_acc_considered':
cfg[cfg.AUTO_MIN_INCR_VAL_ACC]
if cfg[cfg.AUTO_MIN_INCR_VAL_ACC] is not None else 0.0,
'epochs_wait_before_decr':
cfg[cfg.NUM_EPOCHS_WAIT]
if cfg[cfg.NUM_EPOCHS_WAIT] is not None else 5,
'div_lr_by':
cfg[cfg.DIV_LR_BY] if cfg[cfg.DIV_LR_BY] is not None else 2.0
},
'poly': {
'epochs_wait_before_decr':
cfg[cfg.NUM_EPOCHS_WAIT] if cfg[cfg.NUM_EPOCHS_WAIT]
is not None else self.numberOfEpochs / 3,
'final_ep_for_sch':
self.numberOfEpochs
},
'expon': {
'epochs_wait_before_decr':
cfg[cfg.NUM_EPOCHS_WAIT] if cfg[cfg.NUM_EPOCHS_WAIT]
is not None else self.numberOfEpochs / 3,
'final_ep_for_sch':
self.numberOfEpochs,
'lr_to_reach_at_last_ep':
cfg[cfg.EXPON_SCH][0]
if cfg[cfg.EXPON_SCH] is not None else 1.0 / (2**8),
'mom_to_reach_at_last_ep':
cfg[cfg.EXPON_SCH][1]
if cfg[cfg.EXPON_SCH] is not None else 0.9
}
}
# Predefined.
if self.lr_sched_params['type'] == 'predef' and self.lr_sched_params[
'predef']['epochs'] is None:
self.errReqPredLrSch()
# ~~~~~~~~~~~~~~ Augmentation~~~~~~~~~~~~~~
# Image level
self.augm_img_prms_tr = {
'affine': None
} # If var is None, no augm at all.
if cfg[cfg.AUGM_IMG_PRMS_TR] is not None:
self.augm_img_prms_tr['affine'] = AugmenterAffineParams(
cfg[cfg.AUGM_IMG_PRMS_TR]['affine'])
# Patch/Segment level
self.augm_sample_prms_tr = {
'hist_dist': None,
'reflect': None,
'rotate90': None
}
if cfg[cfg.AUGM_SAMPLE_PRMS_TR] is not None:
for key in cfg[cfg.AUGM_SAMPLE_PRMS_TR]:
# For exact form of parameters, see ./deepmedic/dataManagement/augmentation.py
self.augm_sample_prms_tr[key] = cfg[
cfg.AUGM_SAMPLE_PRMS_TR][key]
# ===================VALIDATION========================
self.val_on_samples_during_train = \
cfg[cfg.PERFORM_VAL_SAMPLES] if cfg[cfg.PERFORM_VAL_SAMPLES] is not None else False
if self.lr_sched_params[
'type'] == 'auto' and not self.val_on_samples_during_train:
self.errorAutoRequiresValSamples()
self.val_on_whole_volumes = \
cfg[cfg.PERFORM_VAL_INFERENCE] if cfg[cfg.PERFORM_VAL_INFERENCE] is not None else False
# Input:
if self.val_on_samples_during_train or self.val_on_whole_volumes:
if cfg[cfg.CHANNELS_VAL]:
listOfAListPerChannelWithFilepathsOfAllCasesVal = [
parseAbsFileLinesInList(
getAbsPathEvenIfRelativeIsGiven(
channelConfPath, abs_path_to_cfg))
for channelConfPath in cfg[cfg.CHANNELS_VAL]
]
# [[case1-ch1, case1-ch2], ..., [caseN-ch1, caseN-ch2]]
self.channelsFilepathsVal = \
[list(item) for item in zip(*tuple(listOfAListPerChannelWithFilepathsOfAllCasesVal))]
else:
self.errReqChannsVal()
else:
self.channelsFilepathsVal = []
if self.val_on_samples_during_train:
self.gtLabelsFilepathsVal = \
parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS_VAL], abs_path_to_cfg)) \
if cfg[cfg.GT_LABELS_VAL] is not None else self.errorReqGtLabelsVal()
elif self.val_on_whole_volumes:
self.gtLabelsFilepathsVal = \
parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS_VAL], abs_path_to_cfg)) \
if cfg[ cfg.GT_LABELS_VAL] is not None else []
else: # Dont perform either of the two validations.
self.gtLabelsFilepathsVal = []
# [Optionals]
self.roiMasksFilepathsVal = \
parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(cfg[cfg.ROI_MASKS_VAL], abs_path_to_cfg)) \
if cfg[cfg.ROI_MASKS_VAL] is not None else None # For fast inf.
# ~~~~~Validation on Samples~~~~~~~~
self.n_samples_per_subep_val = \
cfg[cfg.NUM_VAL_SEGMS_LOADED_PERSUB] if cfg[cfg.NUM_VAL_SEGMS_LOADED_PERSUB] is not None else 3000
self.batchsize_val_samples = cfg[cfg.BATCHSIZE_VAL_SAMPL] if cfg[
cfg.BATCHSIZE_VAL_SAMPL] is not None else 50
# ~~~~~~~~~ Sampling (Validation) ~~~~~~~~~~~
sampling_type_flag_val = cfg[cfg.TYPE_OF_SAMPLING_VAL] if cfg[
cfg.TYPE_OF_SAMPLING_VAL] is not None else 1
self.sampling_type_inst_val = samplingType.SamplingType(
self.log, sampling_type_flag_val, num_classes)
if sampling_type_flag_val in [
0, 3
] and cfg[cfg.PROP_OF_SAMPLES_PER_CAT_VAL] is not None:
self.sampling_type_inst_val.set_perc_of_samples_per_cat(
cfg[cfg.PROP_OF_SAMPLES_PER_CAT_VAL])
else:
n_sampl_cats_val = self.sampling_type_inst_val.get_n_sampling_cats(
)
self.sampling_type_inst_val.set_perc_of_samples_per_cat(
[1.0 / n_sampl_cats_val] * n_sampl_cats_val)
self.paths_to_wmaps_per_sampl_cat_per_subj_val = None
if cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_VAL] is not None:
# [[case1-weightMap1, ..., caseN-weightMap1], [case1-weightMap2,...,caseN-weightMap2]]
self.paths_to_wmaps_per_sampl_cat_per_subj_val = [
parseAbsFileLinesInList(
getAbsPathEvenIfRelativeIsGiven(weightMapConfPath,
abs_path_to_cfg)) for
weightMapConfPath in cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_VAL]
]
# ~~~~~~Full inference on validation image~~~~~~
self.num_epochs_between_val_on_whole_volumes = \
cfg[cfg.NUM_EPOCHS_BETWEEN_VAL_INF] if cfg[cfg.NUM_EPOCHS_BETWEEN_VAL_INF] is not None else 1
if self.num_epochs_between_val_on_whole_volumes == 0 and self.val_on_whole_volumes:
self.errorReqNumberOfEpochsBetweenFullValInfGreaterThan0()
self.batchsize_val_whole = cfg[cfg.BATCHSIZE_VAL_WHOLE] if cfg[
cfg.BATCHSIZE_VAL_WHOLE] is not None else 10
# predictions
self.saveSegmentationVal = cfg[cfg.SAVE_SEGM_VAL] if cfg[
cfg.SAVE_SEGM_VAL] is not None else True
self.saveProbMapsBoolPerClassVal = \
cfg[cfg.SAVE_PROBMAPS_PER_CLASS_VAL] \
if (cfg[cfg.SAVE_PROBMAPS_PER_CLASS_VAL] is not None and cfg[cfg.SAVE_PROBMAPS_PER_CLASS_VAL] != []) \
else [True] * num_classes
# Filled by call to self.makeFilepathsForPredictionsAndFeatures()
self.filepathsToSavePredictionsForEachPatientVal = None
self.suffixForSegmAndProbsDictVal = cfg[cfg.SUFFIX_SEGM_PROB_VAL] \
if cfg[cfg.SUFFIX_SEGM_PROB_VAL] is not None \
else {"segm": "Segm", "prob": "ProbMapClass"}
# features:
self.save_fms_flag_val = \
cfg[cfg.SAVE_INDIV_FMS_VAL] if cfg[cfg.SAVE_INDIV_FMS_VAL] is not None else False
if self.save_fms_flag_val is True:
indices_fms_per_pathtype_per_layer_to_save = [cfg[cfg.INDICES_OF_FMS_TO_SAVE_NORMAL_VAL]] + \
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_SUBSAMPLED_VAL]] + \
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_FC_VAL]]
# By default, save none.
self.indices_fms_per_pathtype_per_layer_to_save = \
[item if item is not None else [] for item in indices_fms_per_pathtype_per_layer_to_save]
else:
self.indices_fms_per_pathtype_per_layer_to_save = None
# Filled by call to self.makeFilepathsForPredictionsAndFeatures()
self.filepathsToSaveFeaturesForEachPatientVal = None
# Output:
# Given by the config file, and is then used to fill filepathsToSavePredictionsForEachPatient
# and filepathsToSaveFeaturesForEachPatient.
self.namesToSavePredictionsAndFeaturesVal = \
parseFileLinesInList(
getAbsPathEvenIfRelativeIsGiven(cfg[cfg.NAMES_FOR_PRED_PER_CASE_VAL], abs_path_to_cfg)) \
if cfg[cfg.NAMES_FOR_PRED_PER_CASE_VAL] \
else None # CAREFUL: Here we use a different parsing function!
if not self.namesToSavePredictionsAndFeaturesVal and self.val_on_whole_volumes \
and (self.saveSegmentationVal or True in self.saveProbMapsBoolPerClassVal
or self.save_fms_flag_val):
self.errorRequireNamesOfPredictionsVal()
# ===================== PRE-PROCESSING ======================
# === Data compatibility checks ===
self.run_input_checks = cfg[cfg.RUN_INP_CHECKS] if cfg[
cfg.RUN_INP_CHECKS] is not None else True
# == Padding ==
self.pad_input = cfg[cfg.PAD_INPUT] if cfg[
cfg.PAD_INPUT] is not None else True
# == Normalization ==
norm_zscore_prms = {
'apply_to_all_channels': False, # True/False
'apply_per_channel':
None, # Must be None if above True. Else, List Bool per channel
'cutoff_percents':
None, # None or [low, high], each from 0.0 to 100. Eg [5.,95.]
'cutoff_times_std':
None, # None or [low, high], each positive Float. Eg [3.,3.]
'cutoff_below_mean': False
}
if cfg[cfg.NORM_ZSCORE_PRMS] is not None:
for key in cfg[cfg.NORM_ZSCORE_PRMS]:
norm_zscore_prms[key] = cfg[cfg.NORM_ZSCORE_PRMS][key]
if norm_zscore_prms['apply_to_all_channels'] and norm_zscore_prms[
'apply_per_channel'] is not None:
self.errorIntNormZScoreTwoAppliesGiven()
if norm_zscore_prms['apply_per_channel'] is not None:
assert len(norm_zscore_prms['apply_per_channel']) == len(
cfg[cfg.CHANNELS_TR]) # num channels
# Aggregate params from all types of normalization:
# norm_prms = None : No int normalization will be performed.
# norm_prms['verbose_lvl']: 0: No logging, 1: Type of cutoffs and timing 2: Stats.
self.norm_prms = {
'verbose_lvl':
cfg[cfg.NORM_VERB_LVL]
if cfg[cfg.NORM_VERB_LVL] is not None else 0,
'zscore':
norm_zscore_prms
}
# ============= OTHERS ==========
# Others useful internally or for reporting:
self.numberOfCasesTrain = len(self.channelsFilepathsTrain)
self.numberOfCasesVal = len(self.channelsFilepathsVal)
# ========= HIDDENS =============
# no config allowed for these at the moment:
# Re-weight samples in the cost function *on a per-class basis*: Type of re-weighting and training schedule.
# E.g. to exclude a class, or counter class imbalance.
# "type": string/None, "prms": any/None, "schedule": [ min_epoch, max_epoch ]
# Type, prms combinations: "freq", None || "per_c", [0., 2., 1., ...] (as many as classes)
# "schedule": Constant before epoch [0], linear change towards equal weight (=1) until epoch [1],
# constant equal weights (=1) afterwards.
self.reweight_classes_in_cost = \
cfg[cfg.W_C_IN_COST] \
if cfg[cfg.W_C_IN_COST] is not None \
else {"type": None,
"prms": None,
"schedule": [0, self.numberOfEpochs]
}
if self.reweight_classes_in_cost["type"] == "per_c":
assert len(self.reweight_classes_in_cost["prms"]) == num_classes
self._makeFilepathsForPredictionsAndFeaturesVal(
folderForPredictionsVal, folderForFeaturesVal)
# ====Optimization=====
self.learningRate = cfg[cfg.LRATE] if cfg[
cfg.LRATE] is not None else 0.001
self.optimizerSgd0Adam1Rms2 = cfg[cfg.OPTIMIZER] if cfg[
cfg.OPTIMIZER] is not None else 2
if self.optimizerSgd0Adam1Rms2 == 0:
self.b1Adam = "placeholder"
self.b2Adam = "placeholder"
self.eAdam = "placeholder"
self.rhoRms = "placeholder"
self.eRms = "placeholder"
elif self.optimizerSgd0Adam1Rms2 == 1:
self.b1Adam = cfg[cfg.B1_ADAM] if cfg[
cfg.
B1_ADAM] is not None else 0.9 # default in paper and seems good
self.b2Adam = cfg[cfg.B2_ADAM] if cfg[
cfg.
B2_ADAM] is not None else 0.999 # default in paper and seems good
self.eAdam = cfg[cfg.EPS_ADAM] if cfg[
cfg.EPS_ADAM] is not None else 10**(-8)
self.rhoRms = "placeholder"
self.eRms = "placeholder"
elif self.optimizerSgd0Adam1Rms2 == 2:
self.b1Adam = "placeholder"
self.b2Adam = "placeholder"
self.eAdam = "placeholder"
self.rhoRms = cfg[cfg.RHO_RMS] if cfg[
cfg.
RHO_RMS] is not None else 0.9 # default in paper and seems good
# 1e-6 was the default in the paper, but blew up the gradients in first try. Never tried 1e-5 yet.
self.eRms = cfg[cfg.EPS_RMS] if cfg[
cfg.EPS_RMS] is not None else 10**(-4)
else:
self.errorRequireOptimizer012()
self.classicMom0Nesterov1 = cfg[cfg.MOM_TYPE] if cfg[
cfg.MOM_TYPE] is not None else 1
if self.classicMom0Nesterov1 not in [0, 1]:
self.errorRequireMomentumClass0Nestov1()
self.momNonNormalized0Normalized1 = cfg[cfg.MOM_NORM_NONNORM] if cfg[
cfg.MOM_NORM_NONNORM] is not None else 1
if self.momNonNormalized0Normalized1 not in [0, 1]:
self.errorRequireMomNonNorm0Norm1()
self.momentumValue = cfg[cfg.MOM] if cfg[cfg.MOM] is not None else 0.6
if self.momentumValue < 0. or self.momentumValue > 1:
self.errorRequireMomValueBetween01()
# ==Regularization==
self.L1_reg_weight = cfg[cfg.L1_REG] if cfg[
cfg.L1_REG] is not None else 0.000001
self.L2_reg_weight = cfg[cfg.L2_REG] if cfg[
cfg.L2_REG] is not None else 0.0001
# ============= HIDDENS ==============
# Indices of layers that should not be trained (kept fixed).
layersToFreezePerPathwayType = [
cfg[cfg.LAYERS_TO_FREEZE_NORM], cfg[cfg.LAYERS_TO_FREEZE_SUBS],
cfg[cfg.LAYERS_TO_FREEZE_FC]
]
indicesOfLayersToFreezeNorm = \
[l - 1 for l in layersToFreezePerPathwayType[0]] if layersToFreezePerPathwayType[0] is not None else []
indicesOfLayersToFreezeSubs = \
[l - 1 for l in layersToFreezePerPathwayType[1]] \
if layersToFreezePerPathwayType[1] is not None \
else indicesOfLayersToFreezeNorm
indicesOfLayersToFreezeFc = \
[l - 1 for l in layersToFreezePerPathwayType[2]] if layersToFreezePerPathwayType[2] is not None else []
# Three sublists, one per pathway type: Normal, Subsampled, FC. eg: [[0,1,2],[0,1,2],[]
self.indicesOfLayersPerPathwayTypeToFreeze = [
indicesOfLayersToFreezeNorm, indicesOfLayersToFreezeSubs,
indicesOfLayersToFreezeFc
]
self.losses_and_weights = cfg[cfg.LOSSES_WEIGHTS] if cfg[
cfg.LOSSES_WEIGHTS] is not None else {
"xentr": 1.0,
"iou": None,
"dsc": None
}
assert True in [
self.losses_and_weights[k] is not None
for k in ["xentr", "iou", "dsc"]
]
self._backwards_compat_with_deprecated_cfg(cfg)
"""
def __init__(self,
log,
mainOutputAbsFolder,
folderForSessionCnnModels,
folderForPredictionsVal,
folderForFeaturesVal,
num_classes,
model_name,
cfg):
#Importants for running session.
# From Session:
self.log = log
self.mainOutputAbsFolder = mainOutputAbsFolder
# From Config:
self.sessionName = self.getSessionName( cfg[cfg.SESSION_NAME] )
abs_path_to_cfg = cfg.get_abs_path_to_cfg()
abs_path_to_saved = getAbsPathEvenIfRelativeIsGiven( cfg[cfg.SAVED_MODEL], abs_path_to_cfg ) if cfg[cfg.SAVED_MODEL] is not None else None # Load pretrained model.
self.savedModelFilepath = check_and_adjust_path_to_ckpt( self.log, abs_path_to_saved) if abs_path_to_saved is not None else None
#====================TRAINING==========================
self.filepath_to_save_models = folderForSessionCnnModels + "/" + model_name + "." + self.sessionName
if cfg[cfg.CHANNELS_TR] is None:
self.errReqChansTr()
if cfg[cfg.GT_LABELS_TR] is None:
self.errReqGtTr()
#[[case1-ch1, ..., caseN-ch1], [case1-ch2,...,caseN-ch2]]
listOfAListPerChannelWithFilepathsOfAllCasesTrain = [parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(channelConfPath, abs_path_to_cfg)) for channelConfPath in cfg[cfg.CHANNELS_TR]]
self.channelsFilepathsTrain = [ list(item) for item in zip(*tuple(listOfAListPerChannelWithFilepathsOfAllCasesTrain)) ] #[[case1-ch1, case1-ch2], ..., [caseN-ch1, caseN-ch2]]
self.gtLabelsFilepathsTrain = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS_TR], abs_path_to_cfg) )
#[Optionals]
#~~~~~~~~~Sampling~~~~~~~
self.providedRoiMasksTrain = True if cfg[cfg.ROI_MASKS_TR] else False
self.roiMasksFilepathsTrain = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.ROI_MASKS_TR], abs_path_to_cfg) ) if self.providedRoiMasksTrain else []
if cfg[cfg.PERC_POS_SAMPLES_TR] is not None : #Deprecated. Issue error and ask for correction.
self.errDeprPercPosTr()
#~~~~~~~~~Advanced Sampling~~~~~~~
#ADVANCED CONFIG IS DISABLED HERE IF useDefaultSamplingFromGtAndRoi = True!
self.useDefaultTrainingSamplingFromGtAndRoi = cfg[cfg.DEFAULT_TR_SAMPLING] if cfg[cfg.DEFAULT_TR_SAMPLING] is not None else True
DEFAULT_SAMPLING_TYPE_TR = 0
if self.useDefaultTrainingSamplingFromGtAndRoi :
self.samplingTypeInstanceTrain = samplingType.SamplingType( self.log, DEFAULT_SAMPLING_TYPE_TR, num_classes )
numberOfCategoriesOfSamplesTr = self.samplingTypeInstanceTrain.getNumberOfCategoriesToSample()
self.samplingTypeInstanceTrain.setPercentOfSamplesPerCategoryToSample( [1.0/numberOfCategoriesOfSamplesTr]*numberOfCategoriesOfSamplesTr )
self.forEachSamplingCategory_aListOfFilepathsToWeightMapsOfEachPatientTraining = None
else :
samplingTypeToUseTr = cfg[cfg.TYPE_OF_SAMPLING_TR] if cfg[cfg.TYPE_OF_SAMPLING_TR] is not None else DEFAULT_SAMPLING_TYPE_TR
self.samplingTypeInstanceTrain = samplingType.SamplingType( self.log, samplingTypeToUseTr, num_classes)
if samplingTypeToUseTr in [0,3] and cfg[cfg.PROP_OF_SAMPLES_PER_CAT_TR] :
self.samplingTypeInstanceTrain.setPercentOfSamplesPerCategoryToSample( cfg[cfg.PROP_OF_SAMPLES_PER_CAT_TR] )
else :
numberOfCategoriesOfSamplesTr = self.samplingTypeInstanceTrain.getNumberOfCategoriesToSample()
self.samplingTypeInstanceTrain.setPercentOfSamplesPerCategoryToSample( [1.0/numberOfCategoriesOfSamplesTr]*numberOfCategoriesOfSamplesTr )
# This could be shortened.
if cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_TR] :
#[[case1-weightMap1, ..., caseN-weightMap1], [case1-weightMap2,...,caseN-weightMap2]]
listOfAListPerWeightMapCategoryWithFilepathsOfAllCasesTrain = [parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(weightMapConfPath, abs_path_to_cfg)) for weightMapConfPath in cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_TR]]
else :
listOfAListPerWeightMapCategoryWithFilepathsOfAllCasesTrain = None
self.forEachSamplingCategory_aListOfFilepathsToWeightMapsOfEachPatientTraining = listOfAListPerWeightMapCategoryWithFilepathsOfAllCasesTrain #If None, following bool will turn False.
self.providedWeightMapsToSampleForEachCategoryTraining = True if self.forEachSamplingCategory_aListOfFilepathsToWeightMapsOfEachPatientTraining else False
#~~~~~~~~ Training Cycle ~~~~~~~~~~~
self.numberOfEpochs = cfg[cfg.NUM_EPOCHS] if cfg[cfg.NUM_EPOCHS] is not None else 35
self.numberOfSubepochs = cfg[cfg.NUM_SUBEP] if cfg[cfg.NUM_SUBEP] is not None else 20
self.numOfCasesLoadedPerSubepoch = cfg[cfg.NUM_CASES_LOADED_PERSUB] if cfg[cfg.NUM_CASES_LOADED_PERSUB] is not None else 50
self.segmentsLoadedOnGpuPerSubepochTrain = cfg[cfg.NUM_TR_SEGMS_LOADED_PERSUB] if cfg[cfg.NUM_TR_SEGMS_LOADED_PERSUB] is not None else 1000
#~~~~~~~ Learning Rate Schedule ~~~~~~~~
assert cfg[cfg.LR_SCH_TYPE] in ['stable', 'predef', 'poly', 'auto', 'expon']
self.lr_sched_params = {'type': cfg[cfg.LR_SCH_TYPE] if cfg[cfg.LR_SCH_TYPE] is not None else 'poly',
'predef': { 'epochs': cfg[cfg.PREDEF_SCH],
'div_lr_by': cfg[cfg.DIV_LR_BY] if cfg[cfg.DIV_LR_BY] is not None else 2.0 },
'auto': { 'min_incr_of_val_acc_considered': cfg[cfg.AUTO_MIN_INCR_VAL_ACC] if cfg[cfg.AUTO_MIN_INCR_VAL_ACC] is not None else 0.0,
'epochs_wait_before_decr': cfg[cfg.NUM_EPOCHS_WAIT] if cfg[cfg.NUM_EPOCHS_WAIT] is not None else 5,
'div_lr_by': cfg[cfg.DIV_LR_BY] if cfg[cfg.DIV_LR_BY] is not None else 2.0 },
'poly': { 'epochs_wait_before_decr': cfg[cfg.NUM_EPOCHS_WAIT] if cfg[cfg.NUM_EPOCHS_WAIT] is not None else self.numberOfEpochs/3,
'final_ep_for_sch': self.numberOfEpochs },
'expon': { 'epochs_wait_before_decr': cfg[cfg.NUM_EPOCHS_WAIT] if cfg[cfg.NUM_EPOCHS_WAIT] is not None else self.numberOfEpochs/3,
'final_ep_for_sch': self.numberOfEpochs,
'lr_to_reach_at_last_ep': cfg[cfg.EXPON_SCH][0] if cfg[cfg.EXPON_SCH] is not None else 1.0/(2**(8)),
'mom_to_reach_at_last_ep': cfg[cfg.EXPON_SCH][1] if cfg[cfg.EXPON_SCH] is not None else 0.9 }
}
#Predefined.
if self.lr_sched_params['type'] == 'predef' and self.lr_sched_params['predef']['epochs'] is None :
self.errReqPredLrSch()
#~~~~~~~~~~~~~~ Augmentation~~~~~~~~~~~~~~
self.reflectImagesPerAxis = cfg[cfg.REFL_AUGM_PER_AXIS] if cfg[cfg.REFL_AUGM_PER_AXIS] else [False, False, False]
self.performIntAugm = cfg[cfg.PERF_INT_AUGM_BOOL] if cfg[cfg.PERF_INT_AUGM_BOOL] is not None else False
if self.performIntAugm :
self.sampleIntAugmShiftWithMuAndStd = cfg[cfg.INT_AUGM_SHIF_MUSTD] if cfg[cfg.INT_AUGM_SHIF_MUSTD] else [0.0 , 0.05]
self.sampleIntAugmMultiWithMuAndStd = cfg[cfg.INT_AUGM_MULT_MUSTD] if cfg[cfg.INT_AUGM_MULT_MUSTD] else [1.0 , 0.01]
self.doIntAugm_shiftMuStd_multiMuStd = [True, self.sampleIntAugmShiftWithMuAndStd, self.sampleIntAugmMultiWithMuAndStd]
else :
self.doIntAugm_shiftMuStd_multiMuStd = [False, 'plcholder', [], []]
#===================VALIDATION========================
self.performValidationOnSamplesThroughoutTraining = cfg[cfg.PERFORM_VAL_SAMPLES] if cfg[cfg.PERFORM_VAL_SAMPLES] is not None else False
if self.lr_sched_params['type'] == 'auto' and not self.performValidationOnSamplesThroughoutTraining :
self.errorAutoRequiresValSamples()
self.performFullInferenceOnValidationImagesEveryFewEpochs = cfg[cfg.PERFORM_VAL_INFERENCE] if cfg[cfg.PERFORM_VAL_INFERENCE] is not None else False
#Input:
if self.performValidationOnSamplesThroughoutTraining or self.performFullInferenceOnValidationImagesEveryFewEpochs :
if cfg[cfg.CHANNELS_VAL] :
listOfAListPerChannelWithFilepathsOfAllCasesVal = [parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(channelConfPath, abs_path_to_cfg)) for channelConfPath in cfg[cfg.CHANNELS_VAL]]
#[[case1-ch1, case1-ch2], ..., [caseN-ch1, caseN-ch2]]
self.channelsFilepathsVal = [ list(item) for item in zip(*tuple(listOfAListPerChannelWithFilepathsOfAllCasesVal)) ]
else :
self.errReqChannsVal()
else :
self.channelsFilepathsVal = []
if self.performValidationOnSamplesThroughoutTraining :
self.gtLabelsFilepathsVal = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS_VAL], abs_path_to_cfg) ) if cfg[cfg.GT_LABELS_VAL] is not None else self.errorReqGtLabelsVal()
elif self.performFullInferenceOnValidationImagesEveryFewEpochs :
self.gtLabelsFilepathsVal = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS_VAL], abs_path_to_cfg) ) if cfg[cfg.GT_LABELS_VAL] is not None else []
else : # Dont perform either of the two validations.
self.gtLabelsFilepathsVal = []
self.providedGtVal = True if self.gtLabelsFilepathsVal is not None else False
#[Optionals]
self.providedRoiMasksVal = True if cfg[cfg.ROI_MASKS_VAL] is not None else False #For fast inf.
self.roiMasksFilepathsVal = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.ROI_MASKS_VAL], abs_path_to_cfg) ) if self.providedRoiMasksVal else []
#~~~~~Validation on Samples~~~~~~~~
self.segmentsLoadedOnGpuPerSubepochVal = cfg[cfg.NUM_VAL_SEGMS_LOADED_PERSUB] if cfg[cfg.NUM_VAL_SEGMS_LOADED_PERSUB] is not None else 3000
#~~~~~~~~~Advanced Validation Sampling~~~~~~~~~~~
#ADVANCED OPTION ARE DISABLED IF useDefaultUniformValidationSampling = True!
self.useDefaultUniformValidationSampling = cfg[cfg.DEFAULT_VAL_SAMPLING] if cfg[cfg.DEFAULT_VAL_SAMPLING] is not None else True
DEFAULT_SAMPLING_TYPE_VAL = 1
if self.useDefaultUniformValidationSampling :
self.samplingTypeInstanceVal = samplingType.SamplingType( self.log, DEFAULT_SAMPLING_TYPE_VAL, num_classes )
numberOfCategoriesOfSamplesVal = self.samplingTypeInstanceVal.getNumberOfCategoriesToSample()
self.samplingTypeInstanceVal.setPercentOfSamplesPerCategoryToSample( [1.0/numberOfCategoriesOfSamplesVal]*numberOfCategoriesOfSamplesVal )
self.perSamplingCat_aListOfFilepathsToWeightMapsOfEachCaseVal = None
else :
samplingTypeToUseVal = cfg[cfg.TYPE_OF_SAMPLING_VAL] if cfg[cfg.TYPE_OF_SAMPLING_VAL] is not None else DEFAULT_SAMPLING_TYPE_VAL
self.samplingTypeInstanceVal = samplingType.SamplingType( self.log, samplingTypeToUseVal, num_classes)
if samplingTypeToUseVal in [0,3] and cfg[cfg.PROP_OF_SAMPLES_PER_CAT_VAL] :
self.samplingTypeInstanceVal.setPercentOfSamplesPerCategoryToSample( cfg[cfg.PROP_OF_SAMPLES_PER_CAT_VAL] )
else :
numberOfCategoriesOfSamplesVal = self.samplingTypeInstanceVal.getNumberOfCategoriesToSample()
self.samplingTypeInstanceVal.setPercentOfSamplesPerCategoryToSample( [1.0/numberOfCategoriesOfSamplesVal]*numberOfCategoriesOfSamplesVal )
# TODO: Shorten this
if cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_VAL] :
#[[case1-weightMap1, ..., caseN-weightMap1], [case1-weightMap2,...,caseN-weightMap2]]
self.perSamplingCat_aListOfFilepathsToWeightMapsOfEachCaseVal = [parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(weightMapConfPath, abs_path_to_cfg)) for weightMapConfPath in cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_VAL]]
else :
self.perSamplingCat_aListOfFilepathsToWeightMapsOfEachCaseVal = None
self.providedWeightMapsToSampleForEachCategoryValidation = True if self.perSamplingCat_aListOfFilepathsToWeightMapsOfEachCaseVal else False
#~~~~~~Full inference on validation image~~~~~~
self.numberOfEpochsBetweenFullInferenceOnValImages = cfg[cfg.NUM_EPOCHS_BETWEEN_VAL_INF] if cfg[cfg.NUM_EPOCHS_BETWEEN_VAL_INF] is not None else 1
if self.numberOfEpochsBetweenFullInferenceOnValImages == 0 and self.performFullInferenceOnValidationImagesEveryFewEpochs :
self.errorReqNumberOfEpochsBetweenFullValInfGreaterThan0()
#predictions
self.saveSegmentationVal = cfg[cfg.SAVE_SEGM_VAL] if cfg[cfg.SAVE_SEGM_VAL] is not None else True
self.saveProbMapsBoolPerClassVal = cfg[cfg.SAVE_PROBMAPS_PER_CLASS_VAL] if (cfg[cfg.SAVE_PROBMAPS_PER_CLASS_VAL] is not None and cfg[cfg.SAVE_PROBMAPS_PER_CLASS_VAL] != []) else [True]*num_classes
self.filepathsToSavePredictionsForEachPatientVal = None #Filled by call to self.makeFilepathsForPredictionsAndFeatures()
#features:
self.saveIndividualFmImagesVal = cfg[cfg.SAVE_INDIV_FMS_VAL] if cfg[cfg.SAVE_INDIV_FMS_VAL] is not None else False
self.saveMultidimensionalImageWithAllFmsVal = cfg[cfg.SAVE_4DIM_FMS_VAL] if cfg[cfg.SAVE_4DIM_FMS_VAL] is not None else False
if self.saveIndividualFmImagesVal == True or self.saveMultidimensionalImageWithAllFmsVal == True:
indices_fms_per_pathtype_per_layer_to_save = [cfg[cfg.INDICES_OF_FMS_TO_SAVE_NORMAL_VAL]] +\
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_SUBSAMPLED_VAL]] +\
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_FC_VAL]]
self.indices_fms_per_pathtype_per_layer_to_save = [item if item is not None else [] for item in indices_fms_per_pathtype_per_layer_to_save] #By default, save none.
else:
self.indices_fms_per_pathtype_per_layer_to_save = None
self.filepathsToSaveFeaturesForEachPatientVal = None #Filled by call to self.makeFilepathsForPredictionsAndFeatures()
#Output:
#Given by the config file, and is then used to fill filepathsToSavePredictionsForEachPatient and filepathsToSaveFeaturesForEachPatient.
self.namesToSavePredictionsAndFeaturesVal = parseFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.NAMES_FOR_PRED_PER_CASE_VAL], abs_path_to_cfg) ) if cfg[cfg.NAMES_FOR_PRED_PER_CASE_VAL] else None #CAREFUL: Here we use a different parsing function!
if not self.namesToSavePredictionsAndFeaturesVal and self.performFullInferenceOnValidationImagesEveryFewEpochs and (self.saveSegmentationVal or True in self.saveProbMapsBoolPerClassVal or self.saveIndividualFmImagesVal or self.saveMultidimensionalImageWithAllFmsVal) :
self.errorRequireNamesOfPredictionsVal()
#===================== OTHERS======================
#Preprocessing
self.padInputImagesBool = cfg[cfg.PAD_INPUT] if cfg[cfg.PAD_INPUT] is not None else True
#Others useful internally or for reporting:
self.numberOfCasesTrain = len(self.channelsFilepathsTrain)
self.numberOfCasesVal = len(self.channelsFilepathsVal)
self.run_input_checks = cfg[cfg.RUN_INP_CHECKS] if cfg[cfg.RUN_INP_CHECKS] is not None else True
#HIDDENS, no config allowed for these at the moment:
self.useSameSubChannelsAsSingleScale = True
self.subsampledChannelsFilepathsTrain = "placeholder" #List of Lists with filepaths per patient. Only used when above is False.
self.subsampledChannelsFilepathsVal = "placeholder" #List of Lists with filepaths per patient. Only used when above is False.
# Reweight classes equally before [0], linearly decrease influence till [1], natural freq after.
self.weight_c_in_xentr_and_release_between_eps = [-1, -1] # Give one negative value to disable it
self._makeFilepathsForPredictionsAndFeaturesVal( folderForPredictionsVal, folderForFeaturesVal )
#====Optimization=====
self.learningRate = cfg[cfg.LRATE] if cfg[cfg.LRATE] is not None else 0.001
self.optimizerSgd0Adam1Rms2 = cfg[cfg.OPTIMIZER] if cfg[cfg.OPTIMIZER] is not None else 2
if self.optimizerSgd0Adam1Rms2 == 0 :
self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder";
self.rhoRms = "placeholder"; self.eRms = "placeholder";
elif self.optimizerSgd0Adam1Rms2 == 1 :
self.b1Adam = cfg[cfg.B1_ADAM] if cfg[cfg.B1_ADAM] is not None else 0.9 #default in paper and seems good
self.b2Adam = cfg[cfg.B2_ADAM] if cfg[cfg.B2_ADAM] is not None else 0.999 #default in paper and seems good
self.eAdam = cfg[cfg.EPS_ADAM] if cfg[cfg.EPS_ADAM] is not None else 10**(-8)
self.rhoRms = "placeholder"; self.eRms = "placeholder";
elif self.optimizerSgd0Adam1Rms2 == 2 :
self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder";
self.rhoRms = cfg[cfg.RHO_RMS] if cfg[cfg.RHO_RMS] is not None else 0.9 #default in paper and seems good
self.eRms = cfg[cfg.EPS_RMS] if cfg[cfg.EPS_RMS] is not None else 10**(-4) # 1e-6 was the default in the paper, but blew up the gradients in first try. Never tried 1e-5 yet.
else :
self.errorRequireOptimizer012()
self.classicMom0Nesterov1 = cfg[cfg.MOM_TYPE] if cfg[cfg.MOM_TYPE] is not None else 1
if self.classicMom0Nesterov1 not in [0,1]:
self.errorRequireMomentumClass0Nestov1()
self.momNonNormalized0Normalized1 = cfg[cfg.MOM_NORM_NONNORM] if cfg[cfg.MOM_NORM_NONNORM] is not None else 1
if self.momNonNormalized0Normalized1 not in [0,1] :
self.errorRequireMomNonNorm0Norm1()
self.momentumValue = cfg[cfg.MOM] if cfg[cfg.MOM] is not None else 0.6
if self.momentumValue < 0. or self.momentumValue > 1:
self.errorRequireMomValueBetween01()
#==Regularization==
self.L1_reg_weight = cfg[cfg.L1_REG] if cfg[cfg.L1_REG] is not None else 0.000001
self.L2_reg_weight = cfg[cfg.L2_REG] if cfg[cfg.L2_REG] is not None else 0.0001
#============= HIDDENS ==============
# Indices of layers that should not be trained (kept fixed).
layersToFreezePerPathwayType = [cfg[cfg.LAYERS_TO_FREEZE_NORM],
cfg[cfg.LAYERS_TO_FREEZE_SUBS],
cfg[cfg.LAYERS_TO_FREEZE_FC]]
indicesOfLayersToFreezeNorm = [ l-1 for l in layersToFreezePerPathwayType[0] ] if layersToFreezePerPathwayType[0] is not None else []
indicesOfLayersToFreezeSubs = [ l-1 for l in layersToFreezePerPathwayType[1] ] if layersToFreezePerPathwayType[1] is not None else indicesOfLayersToFreezeNorm
indicesOfLayersToFreezeFc = [ l-1 for l in layersToFreezePerPathwayType[2] ] if layersToFreezePerPathwayType[2] is not None else []
# Three sublists, one per pathway type: Normal, Subsampled, FC. eg: [[0,1,2],[0,1,2],[]
self.indicesOfLayersPerPathwayTypeToFreeze = [ indicesOfLayersToFreezeNorm, indicesOfLayersToFreezeSubs, indicesOfLayersToFreezeFc ]
self.losses_and_weights = cfg[cfg.LOSSES_WEIGHTS] if cfg[cfg.LOSSES_WEIGHTS] is not None else {"xentr": 1.0, "iou": None, "dsc": None}
assert True in [ self.losses_and_weights[k] is not None for k in ["xentr", "iou", "dsc"] ]
"""
def __init__(self,
log,
mainOutputAbsFolder,
folderForSessionCnnModels,
folderForPredictionsVal,
folderForFeaturesVal,
num_classes,
model_name,
cfg):
#Importants for running session.
# From Session:
self.log = log
self.mainOutputAbsFolder = mainOutputAbsFolder
# From Config:
self.sessionName = self.getSessionName( cfg[cfg.SESSION_NAME] )
abs_path_to_cfg = cfg.get_abs_path_to_cfg()
abs_path_to_saved = getAbsPathEvenIfRelativeIsGiven( cfg[cfg.SAVED_MODEL], abs_path_to_cfg ) if cfg[cfg.SAVED_MODEL] is not None else None # Load pretrained model.
self.savedModelFilepath = check_and_adjust_path_to_ckpt( self.log, abs_path_to_saved) if abs_path_to_saved is not None else None
#====================TRAINING==========================
self.filepath_to_save_models = folderForSessionCnnModels + "/" + model_name + "." + self.sessionName
if cfg[cfg.CHANNELS_TR] is None:
self.errReqChansTr()
if cfg[cfg.GT_LABELS_TR] is None:
self.errReqGtTr()
#[[case1-ch1, ..., caseN-ch1], [case1-ch2,...,caseN-ch2]]
listOfAListPerChannelWithFilepathsOfAllCasesTrain = [parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(channelConfPath, abs_path_to_cfg)) for channelConfPath in cfg[cfg.CHANNELS_TR]]
self.channelsFilepathsTrain = [ list(item) for item in zip(*tuple(listOfAListPerChannelWithFilepathsOfAllCasesTrain)) ] #[[case1-ch1, case1-ch2], ..., [caseN-ch1, caseN-ch2]]
self.gtLabelsFilepathsTrain = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS_TR], abs_path_to_cfg) )
#[Optionals]
#~~~~~~~~~Sampling~~~~~~~
self.providedRoiMasksTrain = True if cfg[cfg.ROI_MASKS_TR] else False
self.roiMasksFilepathsTrain = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.ROI_MASKS_TR], abs_path_to_cfg) ) if self.providedRoiMasksTrain else []
samplingTypeToUseTr = cfg[cfg.TYPE_OF_SAMPLING_TR] if cfg[cfg.TYPE_OF_SAMPLING_TR] is not None else 3
self.samplingTypeInstanceTrain = samplingType.SamplingType( self.log, samplingTypeToUseTr, num_classes)
if samplingTypeToUseTr in [0,3] and cfg[cfg.PROP_OF_SAMPLES_PER_CAT_TR] is not None :
self.samplingTypeInstanceTrain.setPercentOfSamplesPerCategoryToSample( cfg[cfg.PROP_OF_SAMPLES_PER_CAT_TR] )
else :
numberOfCategoriesOfSamplesTr = self.samplingTypeInstanceTrain.getNumberOfCategoriesToSample()
self.samplingTypeInstanceTrain.setPercentOfSamplesPerCategoryToSample( [1.0/numberOfCategoriesOfSamplesTr]*numberOfCategoriesOfSamplesTr )
self.paths_to_wmaps_per_sampl_cat_per_subj_train = None
if cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_TR] is not None :
#[[case1-weightMap1, ..., caseN-weightMap1], [case1-weightMap2,...,caseN-weightMap2]]
self.paths_to_wmaps_per_sampl_cat_per_subj_train = [parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(weightMapConfPath, abs_path_to_cfg)) for weightMapConfPath in cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_TR]]
self.providedWeightMapsToSampleForEachCategoryTraining = self.paths_to_wmaps_per_sampl_cat_per_subj_train is not None
#~~~~~~~~ Training Cycle ~~~~~~~~~~~
self.numberOfEpochs = cfg[cfg.NUM_EPOCHS] if cfg[cfg.NUM_EPOCHS] is not None else 35
self.numberOfSubepochs = cfg[cfg.NUM_SUBEP] if cfg[cfg.NUM_SUBEP] is not None else 20
self.max_n_cases_per_subep_train = cfg[cfg.NUM_CASES_LOADED_PERSUB] if cfg[cfg.NUM_CASES_LOADED_PERSUB] is not None else 50
self.n_samples_per_subep_train = cfg[cfg.NUM_TR_SEGMS_LOADED_PERSUB] if cfg[cfg.NUM_TR_SEGMS_LOADED_PERSUB] is not None else 1000
self.batchsize_train = cfg[cfg.BATCHSIZE_TR] if cfg[cfg.BATCHSIZE_TR] is not None else errReqBatchSizeTr()
self.num_parallel_proc_sampling = cfg[cfg.NUM_OF_PROC_SAMPL] if cfg[cfg.NUM_OF_PROC_SAMPL] is not None else 0
#~~~~~~~ Learning Rate Schedule ~~~~~~~~
assert cfg[cfg.LR_SCH_TYPE] in ['stable', 'predef', 'poly', 'auto', 'expon']
self.lr_sched_params = {'type': cfg[cfg.LR_SCH_TYPE] if cfg[cfg.LR_SCH_TYPE] is not None else 'poly',
'predef': { 'epochs': cfg[cfg.PREDEF_SCH],
'div_lr_by': cfg[cfg.DIV_LR_BY] if cfg[cfg.DIV_LR_BY] is not None else 2.0 },
'auto': { 'min_incr_of_val_acc_considered': cfg[cfg.AUTO_MIN_INCR_VAL_ACC] if cfg[cfg.AUTO_MIN_INCR_VAL_ACC] is not None else 0.0,
'epochs_wait_before_decr': cfg[cfg.NUM_EPOCHS_WAIT] if cfg[cfg.NUM_EPOCHS_WAIT] is not None else 5,
'div_lr_by': cfg[cfg.DIV_LR_BY] if cfg[cfg.DIV_LR_BY] is not None else 2.0 },
'poly': { 'epochs_wait_before_decr': cfg[cfg.NUM_EPOCHS_WAIT] if cfg[cfg.NUM_EPOCHS_WAIT] is not None else self.numberOfEpochs/3,
'final_ep_for_sch': self.numberOfEpochs },
'expon': { 'epochs_wait_before_decr': cfg[cfg.NUM_EPOCHS_WAIT] if cfg[cfg.NUM_EPOCHS_WAIT] is not None else self.numberOfEpochs/3,
'final_ep_for_sch': self.numberOfEpochs,
'lr_to_reach_at_last_ep': cfg[cfg.EXPON_SCH][0] if cfg[cfg.EXPON_SCH] is not None else 1.0/(2**(8)),
'mom_to_reach_at_last_ep': cfg[cfg.EXPON_SCH][1] if cfg[cfg.EXPON_SCH] is not None else 0.9 }
}
#Predefined.
if self.lr_sched_params['type'] == 'predef' and self.lr_sched_params['predef']['epochs'] is None :
self.errReqPredLrSch()
#~~~~~~~~~~~~~~ Augmentation~~~~~~~~~~~~~~
self.augm_params_tr = {'hist_dist': None, 'reflect': None, 'rotate90': None}
if cfg[cfg.AUGM_PARAMS_TR] is not None:
for key in cfg[cfg.AUGM_PARAMS_TR]:
self.augm_params_tr[key] = cfg[cfg.AUGM_PARAMS_TR][key] # For exact form of parameters, see ./deepmedic/dataManagement/augmentation.py
#===================VALIDATION========================
self.val_on_samples_during_train = cfg[cfg.PERFORM_VAL_SAMPLES] if cfg[cfg.PERFORM_VAL_SAMPLES] is not None else False
if self.lr_sched_params['type'] == 'auto' and not self.val_on_samples_during_train :
self.errorAutoRequiresValSamples()
self.val_on_whole_volumes = cfg[cfg.PERFORM_VAL_INFERENCE] if cfg[cfg.PERFORM_VAL_INFERENCE] is not None else False
#Input:
if self.val_on_samples_during_train or self.val_on_whole_volumes :
if cfg[cfg.CHANNELS_VAL] :
listOfAListPerChannelWithFilepathsOfAllCasesVal = [parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(channelConfPath, abs_path_to_cfg)) for channelConfPath in cfg[cfg.CHANNELS_VAL]]
#[[case1-ch1, case1-ch2], ..., [caseN-ch1, caseN-ch2]]
self.channelsFilepathsVal = [ list(item) for item in zip(*tuple(listOfAListPerChannelWithFilepathsOfAllCasesVal)) ]
else :
self.errReqChannsVal()
else :
self.channelsFilepathsVal = []
if self.val_on_samples_during_train :
self.gtLabelsFilepathsVal = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS_VAL], abs_path_to_cfg) ) if cfg[cfg.GT_LABELS_VAL] is not None else self.errorReqGtLabelsVal()
elif self.val_on_whole_volumes :
self.gtLabelsFilepathsVal = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.GT_LABELS_VAL], abs_path_to_cfg) ) if cfg[cfg.GT_LABELS_VAL] is not None else []
else : # Dont perform either of the two validations.
self.gtLabelsFilepathsVal = []
self.providedGtVal = True if self.gtLabelsFilepathsVal is not None else False
#[Optionals]
self.providedRoiMasksVal = True if cfg[cfg.ROI_MASKS_VAL] is not None else False #For fast inf.
self.roiMasksFilepathsVal = parseAbsFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.ROI_MASKS_VAL], abs_path_to_cfg) ) if self.providedRoiMasksVal else []
#~~~~~Validation on Samples~~~~~~~~
self.n_samples_per_subep_val = cfg[cfg.NUM_VAL_SEGMS_LOADED_PERSUB] if cfg[cfg.NUM_VAL_SEGMS_LOADED_PERSUB] is not None else 3000
self.batchsize_val_samples = cfg[cfg.BATCHSIZE_VAL_SAMPL] if cfg[cfg.BATCHSIZE_VAL_SAMPL] is not None else 50
#~~~~~~~~~ Sampling (Validation) ~~~~~~~~~~~
samplingTypeToUseVal = cfg[cfg.TYPE_OF_SAMPLING_VAL] if cfg[cfg.TYPE_OF_SAMPLING_VAL] is not None else 1
self.samplingTypeInstanceVal = samplingType.SamplingType( self.log, samplingTypeToUseVal, num_classes)
if samplingTypeToUseVal in [0,3] and cfg[cfg.PROP_OF_SAMPLES_PER_CAT_VAL] is not None:
self.samplingTypeInstanceVal.setPercentOfSamplesPerCategoryToSample( cfg[cfg.PROP_OF_SAMPLES_PER_CAT_VAL] )
else :
numberOfCategoriesOfSamplesVal = self.samplingTypeInstanceVal.getNumberOfCategoriesToSample()
self.samplingTypeInstanceVal.setPercentOfSamplesPerCategoryToSample( [1.0/numberOfCategoriesOfSamplesVal]*numberOfCategoriesOfSamplesVal )
self.paths_to_wmaps_per_sampl_cat_per_subj_val = None
if cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_VAL] is not None:
#[[case1-weightMap1, ..., caseN-weightMap1], [case1-weightMap2,...,caseN-weightMap2]]
self.paths_to_wmaps_per_sampl_cat_per_subj_val = [parseAbsFileLinesInList(getAbsPathEvenIfRelativeIsGiven(weightMapConfPath, abs_path_to_cfg)) for weightMapConfPath in cfg[cfg.WEIGHT_MAPS_PER_CAT_FILEPATHS_VAL]]
self.providedWeightMapsToSampleForEachCategoryValidation = self.paths_to_wmaps_per_sampl_cat_per_subj_val is not None
#~~~~~~Full inference on validation image~~~~~~
self.num_epochs_between_val_on_whole_volumes = cfg[cfg.NUM_EPOCHS_BETWEEN_VAL_INF] if cfg[cfg.NUM_EPOCHS_BETWEEN_VAL_INF] is not None else 1
if self.num_epochs_between_val_on_whole_volumes == 0 and self.val_on_whole_volumes :
self.errorReqNumberOfEpochsBetweenFullValInfGreaterThan0()
self.batchsize_val_whole = cfg[cfg.BATCHSIZE_VAL_WHOLE] if cfg[cfg.BATCHSIZE_VAL_WHOLE] is not None else 10
#predictions
self.saveSegmentationVal = cfg[cfg.SAVE_SEGM_VAL] if cfg[cfg.SAVE_SEGM_VAL] is not None else True
self.saveProbMapsBoolPerClassVal = cfg[cfg.SAVE_PROBMAPS_PER_CLASS_VAL] if (cfg[cfg.SAVE_PROBMAPS_PER_CLASS_VAL] is not None and cfg[cfg.SAVE_PROBMAPS_PER_CLASS_VAL] != []) else [True]*num_classes
self.filepathsToSavePredictionsForEachPatientVal = None #Filled by call to self.makeFilepathsForPredictionsAndFeatures()
self.suffixForSegmAndProbsDictVal = cfg[cfg.SUFFIX_SEGM_PROB_VAL] if cfg[cfg.SUFFIX_SEGM_PROB_VAL] is not None else {"segm": "Segm", "prob": "ProbMapClass"}
#features:
self.saveIndividualFmImagesVal = cfg[cfg.SAVE_INDIV_FMS_VAL] if cfg[cfg.SAVE_INDIV_FMS_VAL] is not None else False
self.saveMultidimensionalImageWithAllFmsVal = cfg[cfg.SAVE_4DIM_FMS_VAL] if cfg[cfg.SAVE_4DIM_FMS_VAL] is not None else False
if self.saveIndividualFmImagesVal == True or self.saveMultidimensionalImageWithAllFmsVal == True:
indices_fms_per_pathtype_per_layer_to_save = [cfg[cfg.INDICES_OF_FMS_TO_SAVE_NORMAL_VAL]] +\
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_SUBSAMPLED_VAL]] +\
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_FC_VAL]]
self.indices_fms_per_pathtype_per_layer_to_save = [item if item is not None else [] for item in indices_fms_per_pathtype_per_layer_to_save] #By default, save none.
else:
self.indices_fms_per_pathtype_per_layer_to_save = None
self.filepathsToSaveFeaturesForEachPatientVal = None #Filled by call to self.makeFilepathsForPredictionsAndFeatures()
#Output:
#Given by the config file, and is then used to fill filepathsToSavePredictionsForEachPatient and filepathsToSaveFeaturesForEachPatient.
self.namesToSavePredictionsAndFeaturesVal = parseFileLinesInList( getAbsPathEvenIfRelativeIsGiven(cfg[cfg.NAMES_FOR_PRED_PER_CASE_VAL], abs_path_to_cfg) ) if cfg[cfg.NAMES_FOR_PRED_PER_CASE_VAL] else None #CAREFUL: Here we use a different parsing function!
if not self.namesToSavePredictionsAndFeaturesVal and self.val_on_whole_volumes and (self.saveSegmentationVal or True in self.saveProbMapsBoolPerClassVal or self.saveIndividualFmImagesVal or self.saveMultidimensionalImageWithAllFmsVal) :
self.errorRequireNamesOfPredictionsVal()
#===================== OTHERS======================
#Preprocessing
self.padInputImagesBool = cfg[cfg.PAD_INPUT] if cfg[cfg.PAD_INPUT] is not None else True
#Others useful internally or for reporting:
self.numberOfCasesTrain = len(self.channelsFilepathsTrain)
self.numberOfCasesVal = len(self.channelsFilepathsVal)
self.run_input_checks = cfg[cfg.RUN_INP_CHECKS] if cfg[cfg.RUN_INP_CHECKS] is not None else True
#HIDDENS, no config allowed for these at the moment:
# Re-weight samples in the cost function *on a per-class basis*: Type of re-weighting and training schedule.
# E.g. to exclude a class, or counter class imbalance.
# "type": string/None, "prms": any/None, "schedule": [ min_epoch, max_epoch ]
# Type, prms combinations: "freq", None || "per_c", [0., 2., 1., ...] (as many as classes)
# "schedule": Constant before epoch [0], linear change towards equal weight (=1) until epoch [1], constant equal weights (=1) afterwards.
self.reweight_classes_in_cost = cfg[cfg.W_C_IN_COST] if cfg[cfg.W_C_IN_COST] is not None else {"type": None, "prms": None, "schedule": [0, self.numberOfEpochs]}
if self.reweight_classes_in_cost["type"] == "per_c":
assert len(self.reweight_classes_in_cost["prms"]) == num_classes
self._makeFilepathsForPredictionsAndFeaturesVal( folderForPredictionsVal, folderForFeaturesVal )
#====Optimization=====
self.learningRate = cfg[cfg.LRATE] if cfg[cfg.LRATE] is not None else 0.001
self.optimizerSgd0Adam1Rms2 = cfg[cfg.OPTIMIZER] if cfg[cfg.OPTIMIZER] is not None else 2
if self.optimizerSgd0Adam1Rms2 == 0 :
self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder";
self.rhoRms = "placeholder"; self.eRms = "placeholder";
elif self.optimizerSgd0Adam1Rms2 == 1 :
self.b1Adam = cfg[cfg.B1_ADAM] if cfg[cfg.B1_ADAM] is not None else 0.9 #default in paper and seems good
self.b2Adam = cfg[cfg.B2_ADAM] if cfg[cfg.B2_ADAM] is not None else 0.999 #default in paper and seems good
self.eAdam = cfg[cfg.EPS_ADAM] if cfg[cfg.EPS_ADAM] is not None else 10**(-8)
self.rhoRms = "placeholder"; self.eRms = "placeholder";
elif self.optimizerSgd0Adam1Rms2 == 2 :
self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder";
self.rhoRms = cfg[cfg.RHO_RMS] if cfg[cfg.RHO_RMS] is not None else 0.9 #default in paper and seems good
self.eRms = cfg[cfg.EPS_RMS] if cfg[cfg.EPS_RMS] is not None else 10**(-4) # 1e-6 was the default in the paper, but blew up the gradients in first try. Never tried 1e-5 yet.
else :
self.errorRequireOptimizer012()
self.classicMom0Nesterov1 = cfg[cfg.MOM_TYPE] if cfg[cfg.MOM_TYPE] is not None else 1
if self.classicMom0Nesterov1 not in [0,1]:
self.errorRequireMomentumClass0Nestov1()
self.momNonNormalized0Normalized1 = cfg[cfg.MOM_NORM_NONNORM] if cfg[cfg.MOM_NORM_NONNORM] is not None else 1
if self.momNonNormalized0Normalized1 not in [0,1] :
self.errorRequireMomNonNorm0Norm1()
self.momentumValue = cfg[cfg.MOM] if cfg[cfg.MOM] is not None else 0.6
if self.momentumValue < 0. or self.momentumValue > 1:
self.errorRequireMomValueBetween01()
#==Regularization==
self.L1_reg_weight = cfg[cfg.L1_REG] if cfg[cfg.L1_REG] is not None else 0.000001
self.L2_reg_weight = cfg[cfg.L2_REG] if cfg[cfg.L2_REG] is not None else 0.0001
#============= HIDDENS ==============
# Indices of layers that should not be trained (kept fixed).
layersToFreezePerPathwayType = [cfg[cfg.LAYERS_TO_FREEZE_NORM],
cfg[cfg.LAYERS_TO_FREEZE_SUBS],
cfg[cfg.LAYERS_TO_FREEZE_FC]]
indicesOfLayersToFreezeNorm = [ l-1 for l in layersToFreezePerPathwayType[0] ] if layersToFreezePerPathwayType[0] is not None else []
indicesOfLayersToFreezeSubs = [ l-1 for l in layersToFreezePerPathwayType[1] ] if layersToFreezePerPathwayType[1] is not None else indicesOfLayersToFreezeNorm
indicesOfLayersToFreezeFc = [ l-1 for l in layersToFreezePerPathwayType[2] ] if layersToFreezePerPathwayType[2] is not None else []
# Three sublists, one per pathway type: Normal, Subsampled, FC. eg: [[0,1,2],[0,1,2],[]
self.indicesOfLayersPerPathwayTypeToFreeze = [ indicesOfLayersToFreezeNorm, indicesOfLayersToFreezeSubs, indicesOfLayersToFreezeFc ]
self.losses_and_weights = cfg[cfg.LOSSES_WEIGHTS] if cfg[cfg.LOSSES_WEIGHTS] is not None else {"xentr": 1.0, "iou": None, "dsc": None}
assert True in [ self.losses_and_weights[k] is not None for k in ["xentr", "iou", "dsc"] ]
self._backwards_compat_with_deprecated_cfg(cfg)
"""
def deepmedic_runner(device_name, model_cfg_file_path, test_cfg_file_path,
train_cfg_file_path, saved_model_path, reset_trainer):
args = argparse.Namespace()
args.device = device_name
args.model_cfg = model_cfg_file_path
args.test_cfg = test_cfg_file_path
args.train_cfg = train_cfg_file_path
args.saved_model = saved_model_path
args.reset_trainer = reset_trainer
cwd = os.getcwd()
#parser = setup_arg_parser()
#args = parser.parse_args()
# if len(sys.argv) == 1:
# print("For help on the usage of this program, please use the option -h."); exit(1)
if not args.model_cfg:
print("ERROR: Option ["+OPT_MODEL+"] must be specified, pointing to a [MODEL_CFG] file that describes the architecture.\n"+\
"Please try [-h] for more information. Exiting.")
exit(1)
if not (args.train_cfg or args.test_cfg):
print("ERROR: One of the options must be specified:\n"+\
"\t["+OPT_TRAIN+"] to start a training session on a model.\n"+\
"\t["+OPT_TEST+"] to test with an existing model.\n"+\
"Please try [-h] for more information. Exiting.")
exit(1)
#Preliminary checks:
if args.test_cfg and args.train_cfg:
print("ERROR:\t["+OPT_TEST+"] cannot be used in conjuction with ["+OPT_TRAIN+"].\n"+\
"\tTo test with an existing network, please just specify a configuration file for the testing process, which will include a path to a trained model, or specify a model with ["+OPT_LOAD+"].. Exiting.")
exit(1)
if args.reset_trainer and not args.train_cfg:
print("ERROR:\tThe option [" + OPT_RESET +
"] can only be used together with the [" + OPT_TRAIN +
"] option.\n\tPlease try -h for more information. Exiting.")
exit(1)
# Parse main files.
if args.model_cfg:
abs_path_model_cfg = getAbsPathEvenIfRelativeIsGiven(
args.model_cfg, cwd)
model_cfg = ModelConfig(abs_path_model_cfg)
# Create session.
if args.train_cfg:
abs_path_train_cfg = getAbsPathEvenIfRelativeIsGiven(
args.train_cfg, cwd)
session = TrainSession(TrainConfig(abs_path_train_cfg))
elif args.test_cfg:
abs_path_test_cfg = getAbsPathEvenIfRelativeIsGiven(args.test_cfg, cwd)
session = TestSession(TestConfig(abs_path_test_cfg))
#Create output folders and logger.
session.make_output_folders()
session.setup_logger()
log = session.get_logger()
log.print3("")
log.print3(
"======================== Starting new session ============================"
)
log.print3("Command line arguments given: \n" + str(args))
check_dev_passed_correctly(args.device)
(sess_device) = set_environment(args.device)
log.print3("Available devices to Tensorflow:\n" +
str(device_lib.list_local_devices()))
try:
#Find out what session we are being asked to perform:
if args.model_cfg: # Should be always true.
log.print3(
"CONFIG: The configuration file for the [model] given is: " +
str(model_cfg.get_abs_path_to_cfg()))
model_params = ModelParameters(log, model_cfg)
model_params.print_params()
# Sessions
log.print3(
"CONFIG: The configuration file for the [session] was loaded from: "
+ str(session.get_abs_path_to_cfg()))
session.override_file_cfg_with_cmd_line_cfg(args)
_ = session.compile_session_params_from_cfg(model_params)
if args.train_cfg:
session.run_session(sess_device, model_params, args.reset_trainer)
elif args.test_cfg:
session.run_session(sess_device, model_params)
# All done.
except (Exception, KeyboardInterrupt) as e:
log.print3("")
log.print3("ERROR: Caught exception from main process: " + str(e))
log.print3(traceback.format_exc())
log.print3("Finished.")
return 'Success'
def __init__(self, log, mainOutputAbsFolder, folderForPredictions,
folderForFeatures, num_classes, cfg):
#Importants for running session.
# From Session:
self.log = log
self.mainOutputAbsFolder = mainOutputAbsFolder
# From test config:
self.sessionName = self.getSessionName(cfg[cfg.SESSION_NAME])
abs_path_to_cfg = cfg.get_abs_path_to_cfg()
abs_path_to_saved = getAbsPathEvenIfRelativeIsGiven(
cfg[cfg.SAVED_MODEL], abs_path_to_cfg
) if cfg[
cfg.
SAVED_MODEL] is not None else None # Where to load the model from.
self.savedModelFilepath = check_and_adjust_path_to_ckpt(
self.log,
abs_path_to_saved) if abs_path_to_saved is not None else None
#Input:
#[[case1-ch1, ..., caseN-ch1], [case1-ch2,...,caseN-ch2]]
listOfAListPerChannelWithFilepathsOfAllCases = [
parseAbsFileLinesInList(
getAbsPathEvenIfRelativeIsGiven(channelConfPath,
abs_path_to_cfg))
for channelConfPath in cfg[cfg.CHANNELS]
]
self.channelsFilepaths = [
list(item) for item in zip(
*tuple(listOfAListPerChannelWithFilepathsOfAllCases))
] # [[case1-ch1, case1-ch2], ..., [caseN-ch1, caseN-ch2]]
self.gtLabelsFilepaths = parseAbsFileLinesInList(
getAbsPathEvenIfRelativeIsGiven(
cfg[cfg.GT_LABELS],
abs_path_to_cfg)) if cfg[cfg.GT_LABELS] is not None else None
self.roiMasksFilepaths = parseAbsFileLinesInList(
getAbsPathEvenIfRelativeIsGiven(
cfg[cfg.ROI_MASKS],
abs_path_to_cfg)) if cfg[cfg.ROI_MASKS] is not None else None
#Output:
self.namesToSavePredictionsAndFeatures = parseFileLinesInList(
getAbsPathEvenIfRelativeIsGiven(cfg[cfg.NAMES_FOR_PRED_PER_CASE],
abs_path_to_cfg)
) if cfg[
cfg.
NAMES_FOR_PRED_PER_CASE] is not None else None #CAREFUL: different parser! #Optional. Not required if not saving results.
#predictions
self.saveSegmentation = cfg[cfg.SAVE_SEGM] if cfg[
cfg.SAVE_SEGM] is not None else True
self.saveProbMapsBoolPerClass = cfg[cfg.SAVE_PROBMAPS_PER_CLASS] if (
cfg[cfg.SAVE_PROBMAPS_PER_CLASS] is not None and
cfg[cfg.SAVE_PROBMAPS_PER_CLASS] != []) else [True] * num_classes
self.filepathsToSavePredictionsForEachPatient = None #Filled by call to self.makeFilepathsForPredictionsAndFeatures()
self.suffixForSegmAndProbsDict = cfg[cfg.SUFFIX_SEGM_PROB] if cfg[
cfg.SUFFIX_SEGM_PROB] is not None else {
"segm": "Segm",
"prob": "ProbMapClass"
}
self.batchsize = cfg[cfg.BATCHSIZE] if cfg[
cfg.BATCHSIZE] is not None else 10
#features:
self.save_fms_flag = cfg[cfg.SAVE_INDIV_FMS] if cfg[
cfg.SAVE_INDIV_FMS] is not None else False
if self.save_fms_flag:
indices_fms_per_pathtype_per_layer_to_save = [cfg[cfg.INDICES_OF_FMS_TO_SAVE_NORMAL]] +\
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_SUBSAMPLED]] +\
[cfg[cfg.INDICES_OF_FMS_TO_SAVE_FC]]
self.indices_fms_per_pathtype_per_layer_to_save = [
item if item is not None else []
for item in indices_fms_per_pathtype_per_layer_to_save
]
else:
self.indices_fms_per_pathtype_per_layer_to_save = None
self.filepathsToSaveFeaturesForEachPatient = None #Filled by call to self.makeFilepathsForPredictionsAndFeatures()
# ===================== PRE-PROCESSING ======================
# === Data compatibility checks ===
self.run_input_checks = cfg[cfg.RUN_INP_CHECKS] if cfg[
cfg.RUN_INP_CHECKS] is not None else True
# == Padding ==
self.pad_input = cfg[cfg.PAD_INPUT] if cfg[
cfg.PAD_INPUT] is not None else True
# == Normalization ==
norm_zscore_prms = {
'apply_to_all_channels': False, # True/False
'apply_per_channel':
None, # Must be None if above True. Else, List Bool per channel
'cutoff_percents':
None, # None or [low, high], each from 0.0 to 100. Eg [5.,95.]
'cutoff_times_std':
None, # None or [low, high], each positive Float. Eg [3.,3.]
'cutoff_below_mean': False
}
if cfg[cfg.NORM_ZSCORE_PRMS] is not None:
for key in cfg[cfg.NORM_ZSCORE_PRMS]:
norm_zscore_prms[key] = cfg[cfg.NORM_ZSCORE_PRMS][key]
if norm_zscore_prms['apply_to_all_channels'] and norm_zscore_prms[
'apply_per_channel'] is not None:
self.errorIntNormZScoreTwoAppliesGiven()
if norm_zscore_prms['apply_per_channel'] is not None:
assert len(norm_zscore_prms['apply_per_channel']) == len(
cfg[cfg.CHANNELS]) # num channels
# Aggregate params from all types of normalization:
# norm_prms = None : No int normalization will be performed.
# norm_prms['verbose_lvl']: 0: No logging, 1: Type of cutoffs and timing 2: Stats.
self.norm_prms = {
'verbose_lvl':
cfg[cfg.NORM_VERB_LVL]
if cfg[cfg.NORM_VERB_LVL] is not None else 0,
'zscore':
norm_zscore_prms
}
# ============= OTHERS =============
#Others useful internally or for reporting:
self.numberOfCases = len(self.channelsFilepaths)
# ============= HIDDENS =============
# no config allowed for these at the moment:
self._makeFilepathsForPredictionsAndFeatures(folderForPredictions,
folderForFeatures)
