def make_cnn_model(
self,
log,
cnnModelName,
#=== Model Parameters ===
numberOfOutputClasses,
numberOfImageChannelsPath1,
numberOfImageChannelsPath2,
#=== Normal Pathway ===
nkerns,
kernelDimensions,
#=== Subsampled Pathway ===
# THESE NEXT TWO, ALONG WITH THE ONES FOR FC, COULD BE PUT IN ONE STRUCTURE WITH NORMAL, EG LIKE kerns = [ [kernsNorm], [kernsSub], [kernsFc]]
nkernsSubsampled, # Used to control if secondary pathways: [] if no secondary pathways. Now its the "factors"
kernelDimensionsSubsampled,
subsampleFactorsPerSubPath, # Controls how many pathways: [] if no secondary pathways. Else, List of lists. One sublist per secondary pathway. Each sublist has 3 ints, the rcz subsampling factors.
#=== FC Layers ===
fcLayersFMs,
kernelDimensionsFirstFcLayer,
softmaxTemperature,
#=== Other Architectural params ===
activationFunc,
#---Residual Connections----
indicesOfLayersToConnectResidualsInOutput,
#--Lower Rank Layer Per Pathway---
indicesOfLowerRankLayersPerPathway,
ranksOfLowerRankLayersForEachPathway,
#---Pooling---
maxPoolingParamsStructure,
#--- Skip Connections --- #Deprecated, not used/supported
convLayersToConnectToFirstFcForMultiscaleFromAllLayerTypes,
#===Size of Image Segments ===
imagePartDimensionsTraining,
imagePartDimensionsValidation,
imagePartDimensionsTesting,
#=== Others ===
# Dropout
dropoutRatesForAllPathways, # list of sublists, one for each pathway. Each either empty or full with the dropout rates of all the layers in the path.
# Initialization
convWInitMethod,
# Batch Normalization
applyBnToInputOfPathways, # one Boolean flag per pathway type. Placeholder for the FC pathway.
movingAvForBnOverXBatches,
):
self.cnnModelName = cnnModelName
# ============= Model Parameters Passed as arguments ================
self.num_classes = numberOfOutputClasses
self.numberOfImageChannelsPath1 = numberOfImageChannelsPath1
self.numberOfImageChannelsPath2 = numberOfImageChannelsPath2
# === Architecture ===
self.nkerns = nkerns # Useless?
self.nkernsSubsampled = nkernsSubsampled # Useless?
self.numSubsPaths = len(
subsampleFactorsPerSubPath
) # do I want this as attribute? Or function is ok?
# fcLayersFMs???
self.kernelDimensionsFirstFcLayer = kernelDimensionsFirstFcLayer
# == Other Architectural Params ==
self.indicesOfLayersToConnectResidualsInOutput = indicesOfLayersToConnectResidualsInOutput
self.indicesOfLowerRankLayersPerPathway = indicesOfLowerRankLayersPerPathway
# pooling?
# == Others ==
self.dropoutRatesForAllPathways = dropoutRatesForAllPathways
# ======== Calculated Attributes =========
#This recField CNN should in future be calculated with all non-secondary pathways, ie normal+fc. Use another variable for pathway.recField.
self.recFieldCnn = calcRecFieldFromKernDimListPerLayerWhenStrides1(
kernelDimensions)
#==============================
rng = np.random.RandomState(seed=None)
######################
# BUILD ACTUAL MODEL #
######################
log.print3("...Building the CNN model...")
# Symbolic variables, which stand for the input. Will be loaded by the compiled trainining/val/test function.
# >>> I should have an input argument. Which, if given None, the below placeholders are created.
if True: # Not given input tensors as arguments
self._setupInputXTensors()
else: # Inputs given as argument tensors. Eg in adv from discr or from batcher.
self._setupInputXTensorsFromGivenArgs(
1, 2, 3, 4, 5, 6
) # Placeholder. Todo: Replace with normal arguments, when input tensor is given. Eg adversarial G.
#=======================Make the NORMAL PATHWAY of the CNN=======================
thisPathway = NormalPathway()
self.pathways.append(thisPathway)
thisPathwayType = thisPathway.pType()
inputToPathwayTrain = self._inp_x['train']['x']
inputToPathwayVal = self._inp_x['val']['x']
inputToPathwayTest = self._inp_x['test']['x']
inputToPathwayShapeTrain = [None, numberOfImageChannelsPath1
] + imagePartDimensionsTraining
inputToPathwayShapeVal = [None, numberOfImageChannelsPath1
] + imagePartDimensionsValidation
inputToPathwayShapeTest = [None, numberOfImageChannelsPath1
] + imagePartDimensionsTesting
thisPathWayNKerns = nkerns
thisPathWayKernelDimensions = kernelDimensions
thisPathwayNumOfLayers = len(thisPathWayNKerns)
thisPathwayUseBnPerLayer = [movingAvForBnOverXBatches > 0
] * thisPathwayNumOfLayers
thisPathwayUseBnPerLayer[0] = applyBnToInputOfPathways[
thisPathwayType] if movingAvForBnOverXBatches > 0 else False # For the 1st layer, ask specific flag.
thisPathwayActivFuncPerLayer = [activationFunc
] * thisPathwayNumOfLayers
thisPathwayActivFuncPerLayer[
0] = "linear" if thisPathwayType != pt.FC else activationFunc # To not apply activation on raw input. -1 is linear activation.
thisPathway.makeLayersOfThisPathwayAndReturnDimensionsOfOutputFM(
log, rng, inputToPathwayTrain, inputToPathwayVal,
inputToPathwayTest, inputToPathwayShapeTrain,
inputToPathwayShapeVal, inputToPathwayShapeTest, thisPathWayNKerns,
thisPathWayKernelDimensions, convWInitMethod,
thisPathwayUseBnPerLayer, movingAvForBnOverXBatches,
thisPathwayActivFuncPerLayer,
dropoutRatesForAllPathways[thisPathwayType],
maxPoolingParamsStructure[thisPathwayType],
indicesOfLowerRankLayersPerPathway[thisPathwayType],
ranksOfLowerRankLayersForEachPathway[thisPathwayType],
indicesOfLayersToConnectResidualsInOutput[thisPathwayType])
[
dimsOfOutputFrom1stPathwayTrain, dimsOfOutputFrom1stPathwayVal,
dimsOfOutputFrom1stPathwayTest
] = thisPathway.getShapeOfOutput()
#=======================Make the SUBSAMPLED PATHWAYs of the CNN=============================
for subpath_i in range(self.numSubsPaths):
thisPathway = SubsampledPathway(
subsampleFactorsPerSubPath[subpath_i])
self.pathways.append(
thisPathway
) # There will be at least an entry as a secondary pathway. But it won't have any layers if it was not actually used.
thisPathwayType = thisPathway.pType()
inputToPathwayTrain = self._inp_x['train']['x_sub_' +
str(subpath_i)]
inputToPathwayVal = self._inp_x['val']['x_sub_' + str(subpath_i)]
inputToPathwayTest = self._inp_x['test']['x_sub_' + str(subpath_i)]
thisPathWayNKerns = nkernsSubsampled[subpath_i]
thisPathWayKernelDimensions = kernelDimensionsSubsampled
thisPathwayNumOfLayers = len(thisPathWayNKerns)
thisPathwayUseBnPerLayer = [movingAvForBnOverXBatches > 0
] * thisPathwayNumOfLayers
thisPathwayUseBnPerLayer[0] = applyBnToInputOfPathways[
thisPathwayType] if movingAvForBnOverXBatches > 0 else False # For the 1st layer, ask specific flag.
thisPathwayActivFuncPerLayer = [activationFunc
] * thisPathwayNumOfLayers
thisPathwayActivFuncPerLayer[
0] = "linear" if thisPathwayType != pt.FC else activationFunc # To not apply activation on raw input. -1 is linear activation.
inputToPathwayShapeTrain = [
None, numberOfImageChannelsPath2
] + thisPathway.calcInputRczDimsToProduceOutputFmsOfCompatibleDims(
thisPathWayKernelDimensions, dimsOfOutputFrom1stPathwayTrain)
inputToPathwayShapeVal = [
None, numberOfImageChannelsPath2
] + thisPathway.calcInputRczDimsToProduceOutputFmsOfCompatibleDims(
thisPathWayKernelDimensions, dimsOfOutputFrom1stPathwayVal)
inputToPathwayShapeTest = [
None, numberOfImageChannelsPath2
] + thisPathway.calcInputRczDimsToProduceOutputFmsOfCompatibleDims(
thisPathWayKernelDimensions, dimsOfOutputFrom1stPathwayTest)
thisPathway.makeLayersOfThisPathwayAndReturnDimensionsOfOutputFM(
log, rng, inputToPathwayTrain, inputToPathwayVal,
inputToPathwayTest, inputToPathwayShapeTrain,
inputToPathwayShapeVal, inputToPathwayShapeTest,
thisPathWayNKerns, thisPathWayKernelDimensions,
convWInitMethod, thisPathwayUseBnPerLayer,
movingAvForBnOverXBatches, thisPathwayActivFuncPerLayer,
dropoutRatesForAllPathways[thisPathwayType],
maxPoolingParamsStructure[thisPathwayType],
indicesOfLowerRankLayersPerPathway[thisPathwayType],
ranksOfLowerRankLayersForEachPathway[thisPathwayType],
indicesOfLayersToConnectResidualsInOutput[thisPathwayType])
# this creates essentially the "upsampling layer"
thisPathway.upsampleOutputToNormalRes(
upsamplingScheme="repeat",
shapeToMatchInRczTrain=dimsOfOutputFrom1stPathwayTrain,
shapeToMatchInRczVal=dimsOfOutputFrom1stPathwayVal,
shapeToMatchInRczTest=dimsOfOutputFrom1stPathwayTest)
#====================================CONCATENATE the output of the 2 cnn-pathways=============================
inputToFirstFcLayerTrain = None
inputToFirstFcLayerVal = None
inputToFirstFcLayerTest = None
numberOfFmsOfInputToFirstFcLayer = 0
for path_i in range(len(self.pathways)):
[
outputNormResOfPathTrain, outputNormResOfPathVal,
outputNormResOfPathTest
] = self.pathways[path_i].getOutputAtNormalRes()
[
dimsOfOutputNormResOfPathTrain, dimsOfOutputNormResOfPathVal,
dimsOfOutputNormResOfPathTest
] = self.pathways[path_i].getShapeOfOutputAtNormalRes()
inputToFirstFcLayerTrain = tf.concat(
[inputToFirstFcLayerTrain, outputNormResOfPathTrain],
axis=1) if path_i != 0 else outputNormResOfPathTrain
inputToFirstFcLayerVal = tf.concat(
[inputToFirstFcLayerVal, outputNormResOfPathVal],
axis=1) if path_i != 0 else outputNormResOfPathVal
inputToFirstFcLayerTest = tf.concat(
[inputToFirstFcLayerTest, outputNormResOfPathTest],
axis=1) if path_i != 0 else outputNormResOfPathTest
numberOfFmsOfInputToFirstFcLayer += dimsOfOutputNormResOfPathTrain[
1]
#======================= Make the Fully Connected Layers =======================
thisPathway = FcPathway()
self.pathways.append(thisPathway)
thisPathwayType = thisPathway.pType()
# This is the shape of the kernel in the first FC layer.
# NOTE: If there is no hidden FC layer, this kernel is used in the Classification layer then.
# Originally it was 1x1x1 only. The pathways themselves where taking care of the receptive field.
# However I can now define it larger (eg 3x3x3), in case it helps combining the multiresolution features better/smoother.
# The convolution is seamless, ie same shape output/input, by mirror padding the input.
firstFcLayerAfterConcatenationKernelShape = self.kernelDimensionsFirstFcLayer
voxelsToPadPerDim = [
kernelDim - 1
for kernelDim in firstFcLayerAfterConcatenationKernelShape
]
log.print3("DEBUG: Shape of the kernel of the first FC layer is : " +
str(firstFcLayerAfterConcatenationKernelShape))
log.print3(
"DEBUG: Input to the FC Pathway will be padded by that many voxels per dimension: "
+ str(voxelsToPadPerDim))
inputToPathwayTrain = padImageWithMirroring(inputToFirstFcLayerTrain,
voxelsToPadPerDim)
inputToPathwayVal = padImageWithMirroring(inputToFirstFcLayerVal,
voxelsToPadPerDim)
inputToPathwayTest = padImageWithMirroring(inputToFirstFcLayerTest,
voxelsToPadPerDim)
inputToPathwayShapeTrain = [None, numberOfFmsOfInputToFirstFcLayer
] + dimsOfOutputFrom1stPathwayTrain[2:5]
inputToPathwayShapeVal = [None, numberOfFmsOfInputToFirstFcLayer
] + dimsOfOutputFrom1stPathwayVal[2:5]
inputToPathwayShapeTest = [None, numberOfFmsOfInputToFirstFcLayer
] + dimsOfOutputFrom1stPathwayTest[2:5]
for rcz_i in range(3):
inputToPathwayShapeTrain[2 + rcz_i] += voxelsToPadPerDim[rcz_i]
inputToPathwayShapeVal[2 + rcz_i] += voxelsToPadPerDim[rcz_i]
inputToPathwayShapeTest[2 + rcz_i] += voxelsToPadPerDim[rcz_i]
thisPathWayNKerns = fcLayersFMs + [self.num_classes]
thisPathWayKernelDimensions = [
firstFcLayerAfterConcatenationKernelShape
] + [[1, 1, 1]] * (len(thisPathWayNKerns) - 1)
thisPathwayNumOfLayers = len(thisPathWayNKerns)
thisPathwayUseBnPerLayer = [movingAvForBnOverXBatches > 0
] * thisPathwayNumOfLayers
thisPathwayUseBnPerLayer[0] = applyBnToInputOfPathways[
thisPathwayType] if movingAvForBnOverXBatches > 0 else False # For the 1st layer, ask specific flag.
thisPathwayActivFuncPerLayer = [activationFunc
] * thisPathwayNumOfLayers
thisPathwayActivFuncPerLayer[
0] = "linear" if thisPathwayType != pt.FC else activationFunc # To not apply activation on raw input. -1 is linear activation.
thisPathway.makeLayersOfThisPathwayAndReturnDimensionsOfOutputFM(
log, rng, inputToPathwayTrain, inputToPathwayVal,
inputToPathwayTest, inputToPathwayShapeTrain,
inputToPathwayShapeVal, inputToPathwayShapeTest, thisPathWayNKerns,
thisPathWayKernelDimensions, convWInitMethod,
thisPathwayUseBnPerLayer, movingAvForBnOverXBatches,
thisPathwayActivFuncPerLayer,
dropoutRatesForAllPathways[thisPathwayType],
maxPoolingParamsStructure[thisPathwayType],
indicesOfLowerRankLayersPerPathway[thisPathwayType],
ranksOfLowerRankLayersForEachPathway[thisPathwayType],
indicesOfLayersToConnectResidualsInOutput[thisPathwayType])
# =========== Make the final Target Layer (softmax, regression, whatever) ==========
log.print3("Adding the final Softmax Target layer...")
self.finalTargetLayer = self._getClassificationLayer()
self.finalTargetLayer.makeLayer(rng,
self.getFcPathway().getLayer(-1),
softmaxTemperature)
(self._output_gt_tensor_feeds['train']['y_gt'],
self._output_gt_tensor_feeds['val']['y_gt']
) = self.finalTargetLayer.get_output_gt_tensor_feed()
log.print3("Finished building the CNN's model.")
def calcRecFieldOfPathway(self, kernelDimsPerLayer):
return calcRecFieldFromKernDimListPerLayerWhenStrides1(
kernelDimsPerLayer)
def __init__(self, log, cfg):
self.log = log
self.cnnModelName = cfg[cfg.MODEL_NAME] if cfg[
cfg.MODEL_NAME] is not None else getDefaultModelName()
#===========MODEL PARAMETERS==========
self.numberClasses = cfg[cfg.NUM_CLASSES] if cfg[
cfg.NUM_CLASSES] is not None else self.errReqNumClasses()
self.numberOfInputChannelsNormal = cfg[cfg.NUM_INPUT_CHANS] if cfg[
cfg.NUM_INPUT_CHANS] is not None else self.errReqNumChannels()
assert self.numberOfInputChannelsNormal > 0
#===Normal pathway===
self.numFMsPerLayerNormal = cfg[
cfg.N_FMS_NORM] if cfg[cfg.N_FMS_NORM] is not None and len(
cfg[cfg.N_FMS_NORM]) > 0 else self.errReqFMsNormal()
numOfLayers = len(self.numFMsPerLayerNormal)
self.kernDimPerLayerNormal = cfg[
cfg.KERN_DIM_NORM] if checkKernDimPerLayerCorrect3dAndNumLayers(
cfg[cfg.KERN_DIM_NORM],
numOfLayers) else self.errReqKernDimNormal()
self.receptiveFieldNormal = calcRecFieldFromKernDimListPerLayerWhenStrides1(
self.kernDimPerLayerNormal) # Just for COMPATIBILITY CHECKS!
residConnAtLayersNormal = cfg[cfg.RESID_CONN_LAYERS_NORM] if cfg[
cfg.RESID_CONN_LAYERS_NORM] is not None else [
] #layer number, starting from 1 for 1st layer. NOT indices.
lowerRankLayersNormal = cfg[cfg.LOWER_RANK_LAYERS_NORM] if cfg[
cfg.LOWER_RANK_LAYERS_NORM] is not None else [
] #layer number, starting from 1 for 1st layer. NOT indices.
#==Subsampled pathway==
self.useSubsampledBool = cfg[cfg.USE_SUBSAMPLED] if cfg[
cfg.USE_SUBSAMPLED] is not None else False
if not self.useSubsampledBool:
self.numFMsPerLayerSubsampled = []
self.kernDimPerLayerSubsampled = []
self.receptiveFieldSubsampled = []
self.subsampleFactor = []
residConnAtLayersSubsampled = []
lowerRankLayersSubsampled = []
else:
self.numFMsPerLayerSubsampled = cfg[cfg.N_FMS_SUBS] if cfg[
cfg.N_FMS_SUBS] is not None else self.numFMsPerLayerNormal
self.numFMsPerLayerSubsampled = self.changeDatastructureToListOfListsForSecondaryPathwaysIfNeeded(
self.numFMsPerLayerSubsampled)
# check that all subsampled pathways have the same number of layers. Limitation in the code currently, because I use kernDimSubsampled for all of them.
if not self.checkThatSublistsHaveSameLength(
self.numFMsPerLayerSubsampled):
self.errReqSameNumOfLayersPerSubPathway()
numOfLayersInEachSubPath = len(self.numFMsPerLayerSubsampled[0])
if cfg[cfg.
KERN_DIM_SUBS] == None and numOfLayersInEachSubPath == len(
self.numFMsPerLayerNormal):
self.kernDimPerLayerSubsampled = self.kernDimPerLayerNormal
self.receptiveFieldSubsampled = self.receptiveFieldNormal
elif cfg[
cfg.
KERN_DIM_SUBS] == None and numOfLayersInEachSubPath != len(
self.numFMsPerLayerNormal
): #user specified subsampled layers.
self.errorRequireKernelDimensionsSubsampled(
self.kernDimPerLayerNormal, cfg[cfg.N_FMS_SUBS])
# kernDimSubsampled was specified. Now it's going to be tricky to make sure everything alright.
elif not checkKernDimPerLayerCorrect3dAndNumLayers(
cfg[cfg.KERN_DIM_SUBS], numOfLayersInEachSubPath):
self.errReqKernDimNormalCorr()
else: #kernel dimensions specified and are correct (3d, same number of layers as subsampled specified). Need to check the two receptive fields and make sure they are correct.
self.kernDimPerLayerSubsampled = cfg[cfg.KERN_DIM_SUBS]
self.receptiveFieldSubsampled = calcRecFieldFromKernDimListPerLayerWhenStrides1(
self.kernDimPerLayerSubsampled)
if self.receptiveFieldNormal != self.receptiveFieldSubsampled:
self.errorReceptiveFieldsOfNormalAndSubsampledDifferent(
self.receptiveFieldNormal,
self.receptiveFieldSubsampled)
#Everything alright, finally. Proceed safely...
self.subsampleFactor = cfg[cfg.SUBS_FACTOR] if cfg[
cfg.SUBS_FACTOR] is not None else [3, 3, 3]
self.subsampleFactor = self.changeDatastructureToListOfListsForSecondaryPathwaysIfNeeded(
self.subsampleFactor)
for secondaryPathway_i in range(
len(self.subsampleFactor)
): #It should now be a list of lists, one sublist per secondary pathway. This is what is currently defining how many pathways to use.
if len(self.subsampleFactor[secondaryPathway_i]) != 3:
self.errorSubFactor3d()
if not checkSubsampleFactorEven(
self.subsampleFactor[secondaryPathway_i]):
self.warnSubFactorOdd()
#---For multiple lower-scale pathways, via the numFMsPerLayerSubsampled and subsampleFactor config ----
numOfSubsPaths = max(len(self.numFMsPerLayerSubsampled),
len(self.subsampleFactor))
# Default behaviour:
# If less sublists in numFMsPerLayerSubsampled were given than numOfSubsPaths, add more sublists of numFMsPerLayerSubsampled, for the extra subpaths.
for _ in range(numOfSubsPaths -
len(self.numFMsPerLayerSubsampled)):
numFmsForLayersOfLastSubPath = self.numFMsPerLayerSubsampled[
-1]
self.numFMsPerLayerSubsampled.append([
max(1, int(numFmsInLayer_i))
for numFmsInLayer_i in numFmsForLayersOfLastSubPath
])
# If less sublists in subsampleFactor were given than numOfSubsPaths, add more sublists of subsampleFactors, for the extra subpaths.
for _ in range(numOfSubsPaths - len(self.subsampleFactor)):
self.subsampleFactor.append(
[
subFactorInDim_i + 2
for subFactorInDim_i in self.subsampleFactor[-1]
]
) # Adds one more sublist, eg [5,5,5], which is the last subFactor, increased by +2 in all rcz dimensions.
# Residuals and lower ranks.
residConnAtLayersSubsampled = cfg[cfg.RESID_CONN_LAYERS_SUBS] if cfg[
cfg.
RESID_CONN_LAYERS_SUBS] is not None else residConnAtLayersNormal
lowerRankLayersSubsampled = cfg[cfg.LOWER_RANK_LAYERS_SUBS] if cfg[
cfg.
LOWER_RANK_LAYERS_SUBS] is not None else lowerRankLayersNormal
#==FC Layers==
self.numFMsInExtraFcs = cfg[cfg.N_FMS_FC] if cfg[
cfg.N_FMS_FC] is not None else []
self.kernelDimensionsFirstFcLayer = cfg[cfg.KERN_DIM_1ST_FC] if cfg[
cfg.KERN_DIM_1ST_FC] is not None else [1, 1, 1]
assert len(self.kernelDimensionsFirstFcLayer) == 3 and (False not in [
dim > 0 for dim in self.kernelDimensionsFirstFcLayer
])
residConnAtLayersFc = cfg[cfg.RESID_CONN_LAYERS_FC] if cfg[
cfg.RESID_CONN_LAYERS_FC] is not None else []
#==Size of Image Segments ==
self.segmDimNormalTrain = cfg[cfg.SEG_DIM_TRAIN] if cfg[
cfg.SEG_DIM_TRAIN] is not None else self.errReqSegmDimTrain()
self.segmDimNormalVal = cfg[cfg.SEG_DIM_VAL] if cfg[
cfg.SEG_DIM_VAL] is not None else self.receptiveFieldNormal
self.segmDimNormalInfer = cfg[cfg.SEG_DIM_INFER] if cfg[
cfg.SEG_DIM_INFER] is not None else self.segmDimNormalTrain
for (tr0_val1_inf2,
segmentDimensions) in [(0, self.segmDimNormalTrain),
(1, self.segmDimNormalVal),
(2, self.segmDimNormalInfer)]:
if not checkRecFieldVsSegmSize(self.receptiveFieldNormal,
segmentDimensions):
self.errorSegmDimensionsSmallerThanReceptiveF(
self.receptiveFieldNormal, segmentDimensions,
tr0_val1_inf2)
#=== Batch Sizes ===
self.batchSizeTrain = cfg[cfg.BATCH_SIZE_TR] if cfg[
cfg.BATCH_SIZE_TR] is not None else self.errReqBatchSizeTr()
self.batchSizeVal = cfg[cfg.BATCH_SIZE_VAL] if cfg[
cfg.BATCH_SIZE_VAL] is not None else self.batchSizeTrain
self.batchSizeInfer = cfg[cfg.BATCH_SIZE_INFER] if cfg[
cfg.BATCH_SIZE_INFER] is not None else self.batchSizeTrain
#=== Dropout rates ===
self.dropNormal = cfg[cfg.DROP_NORM] if cfg[
cfg.DROP_NORM] is not None else []
self.dropSubsampled = cfg[cfg.DROP_SUBS] if cfg[
cfg.DROP_SUBS] is not None else []
self.dropFc = cfg[cfg.DROP_FC] if cfg[
cfg.DROP_FC] is not None else self.defaultDropFcList(
self.numFMsInExtraFcs) #default = [0.0, 0.5, ..., 0.5]
self.dropoutRatesForAllPathways = [
self.dropNormal, self.dropSubsampled, self.dropFc, []
]
#== Weight Initialization==
self.convWInitMethod = cfg[cfg.CONV_W_INIT] if cfg[
cfg.CONV_W_INIT] is not None else ["fanIn", 2]
if not self.convWInitMethod[0] in ["normal", "fanIn"]:
self.errorReqInitializationMethod()
#== Activation Function ==
self.activationFunc = cfg[cfg.ACTIV_FUNC] if cfg[
cfg.ACTIV_FUNC] is not None else "prelu"
if not self.activationFunc in [
"linear", "relu", "prelu", "elu", "selu"
]:
self.errorReqActivFunction()
#==BATCH NORMALIZATION==
self.applyBnToInputOfPathways = [
False, False, True
] # Per pathway type. The 3rd entry, for FC, should always be True.
self.bnRollAverOverThatManyBatches = cfg[
cfg.BN_ROLL_AV_BATCHES] if cfg[
cfg.BN_ROLL_AV_BATCHES] is not None else 60
#==============CALCULATED=====================
# Residual Connections backwards, per pathway type :
self.checkLayersForResidualsGivenDoNotInclude1st(
residConnAtLayersNormal, residConnAtLayersSubsampled,
residConnAtLayersFc)
# The following variable passed to the system takes indices, ie number starts from 0. User specifies from 1.
self.indicesOfLayersToConnectResidualsInOutput = [
[layerNum - 1 for layerNum in residConnAtLayersNormal],
[layerNum - 1 for layerNum in residConnAtLayersSubsampled],
[layerNum - 1 for layerNum in residConnAtLayersFc], []
]
self.indicesOfLowerRankLayersPerPathway = [
[layerNum - 1 for layerNum in lowerRankLayersNormal],
[layerNum - 1 for layerNum in lowerRankLayersSubsampled],
[], #FC doesn't make sense to be lower rank. It's 1x1x1 anyway.
[]
]
self.ranksOfLowerRankLayersForEachPathway = [[
2 for layer_i in self.indicesOfLowerRankLayersPerPathway[0]
], [2 for layer_i in self.indicesOfLowerRankLayersPerPathway[1]], [],
[]]
#============= HIDDENS ======================
self.numberOfInputChannelsSubsampled = self.numberOfInputChannelsNormal
#MultiscaleConnections:
self.convLayersToConnectToFirstFcForMultiscaleFromAllLayerTypes = [
[], []
] #a sublist for each pathway. Starts from 0 index. Give a sublist, even empty for no connections.
#... It's ok if I dont have a 2nd path but still give a 2nd sublist, it's controlled by nkernsSubsampled.
#-------POOLING---------- (not fully supported currently)
#One entry per pathway-type. leave [] if the pathway does not exist or there is no mp there AT ALL.
#Inside each entry, put a list FOR EACH LAYER. It should be [] for the layer if no mp there. But FOR EACH LAYER.
#MP is applied >>AT THE INPUT of the layer<<. To use mp to a layer, put a list of [[dsr,dsc,dsz], [strr,strc,strz], [mirrorPad-r,-c,-z], mode] which give the dimensions of the mp window, the stride, how many times to mirror the last slot at each dimension for padding (give 0 for none), the mode (usually 'max' pool). Eg [[2,2,2],[1,1,1]] or [[2,2,2],[2,2,2]] usually.
#If a pathway is not used (eg subsampled), put an empty list in the first dimension entry.
mpParamsNorm = [
[] for layeri in range(len(self.numFMsPerLayerNormal))
] #[[[2,2,2], [1,1,1], [1,1,1], 'MAX'], [],[],[],[],[],[], []], #first pathway
mpParamsSubs = [
[] for layeri in range(len(self.numFMsPerLayerSubsampled[0]))
] if self.useSubsampledBool else [
] # CAREFUL about the [0]. Only here till this structure is made different per pathway and not pathwayType.
mpParamsFc = [
[] for layeri in range(len(self.numFMsInExtraFcs) + 1)
] #FC. This should NEVER be used for segmentation. Possible for classification though.
self.maxPoolingParamsStructure = [
mpParamsNorm, mpParamsSubs, mpParamsFc
]
self.softmaxTemperature = 1.0 #Higher temperatures make the probabilities LESS distinctable. Actions have more similar probabilities.
def make_cnn_model( self,
log,
cnnModelName,
#=== Model Parameters ===
numberOfOutputClasses,
numberOfImageChannelsPath1,
numberOfImageChannelsPath2,
#=== Normal Pathway ===
nkerns,
kernelDimensions,
#=== Subsampled Pathway ===
# THESE NEXT TWO, ALONG WITH THE ONES FOR FC, COULD BE PUT IN ONE STRUCTURE WITH NORMAL, EG LIKE kerns = [ [kernsNorm], [kernsSub], [kernsFc]]
nkernsSubsampled, # Used to control if secondary pathways: [] if no secondary pathways. Now its the "factors"
kernelDimensionsSubsampled,
subsampleFactorsPerSubPath, # Controls how many pathways: [] if no secondary pathways. Else, List of lists. One sublist per secondary pathway. Each sublist has 3 ints, the rcz subsampling factors.
#=== FC Layers ===
fcLayersFMs,
kernelDimensionsFirstFcLayer,
softmaxTemperature,
#=== Other Architectural params ===
activationFunc,
#---Residual Connections----
indicesOfLayersToConnectResidualsInOutput,
#--Lower Rank Layer Per Pathway---
indicesOfLowerRankLayersPerPathway,
ranksOfLowerRankLayersForEachPathway,
#---Pooling---
maxPoolingParamsStructure,
#--- Skip Connections --- #Deprecated, not used/supported
convLayersToConnectToFirstFcForMultiscaleFromAllLayerTypes,
#===Size of Image Segments ===
imagePartDimensionsTraining ,
imagePartDimensionsValidation,
imagePartDimensionsTesting,
#=== Others ===
# Dropout
dropoutRatesForAllPathways, # list of sublists, one for each pathway. Each either empty or full with the dropout rates of all the layers in the path.
# Initialization
convWInitMethod,
# Batch Normalization
applyBnToInputOfPathways, # one Boolean flag per pathway type. Placeholder for the FC pathway.
movingAvForBnOverXBatches,
):
self.cnnModelName = cnnModelName
# ============= Model Parameters Passed as arguments ================
self.num_classes = numberOfOutputClasses
self.numberOfImageChannelsPath1 = numberOfImageChannelsPath1
self.numberOfImageChannelsPath2 = numberOfImageChannelsPath2
# === Architecture ===
self.nkerns = nkerns # Useless?
self.nkernsSubsampled = nkernsSubsampled # Useless?
self.numSubsPaths = len(subsampleFactorsPerSubPath) # do I want this as attribute? Or function is ok?
# fcLayersFMs???
self.kernelDimensionsFirstFcLayer = kernelDimensionsFirstFcLayer
# == Other Architectural Params ==
self.indicesOfLayersToConnectResidualsInOutput = indicesOfLayersToConnectResidualsInOutput
self.indicesOfLowerRankLayersPerPathway = indicesOfLowerRankLayersPerPathway
# pooling?
# == Others ==
self.dropoutRatesForAllPathways = dropoutRatesForAllPathways
# ======== Calculated Attributes =========
#This recField CNN should in future be calculated with all non-secondary pathways, ie normal+fc. Use another variable for pathway.recField.
self.recFieldCnn = calcRecFieldFromKernDimListPerLayerWhenStrides1(kernelDimensions)
#==============================
rng = np.random.RandomState(seed=None)
######################
# BUILD ACTUAL MODEL #
######################
log.print3("...Building the CNN model...")
# Symbolic variables, which stand for the input. Will be loaded by the compiled trainining/val/test function.
# >>> I should have an input argument. Which, if given None, the below placeholders are created.
if True: # Not given input tensors as arguments
self._setupInputXTensors()
else: # Inputs given as argument tensors. Eg in adv from discr or from batcher.
self._setupInputXTensorsFromGivenArgs(1,2,3,4,5,6) # Placeholder. Todo: Replace with normal arguments, when input tensor is given. Eg adversarial G.
#=======================Make the NORMAL PATHWAY of the CNN=======================
thisPathway = NormalPathway()
self.pathways.append(thisPathway)
thisPathwayType = thisPathway.pType()
inputToPathwayTrain = self._inp_x['train']['x']
inputToPathwayVal = self._inp_x['val']['x']
inputToPathwayTest = self._inp_x['test']['x']
inputToPathwayShapeTrain = [None, numberOfImageChannelsPath1] + imagePartDimensionsTraining
inputToPathwayShapeVal = [None, numberOfImageChannelsPath1] + imagePartDimensionsValidation
inputToPathwayShapeTest = [None, numberOfImageChannelsPath1] + imagePartDimensionsTesting
thisPathWayNKerns = nkerns
thisPathWayKernelDimensions = kernelDimensions
thisPathwayNumOfLayers = len(thisPathWayNKerns)
thisPathwayUseBnPerLayer = [movingAvForBnOverXBatches > 0] * thisPathwayNumOfLayers
thisPathwayUseBnPerLayer[0] = applyBnToInputOfPathways[thisPathwayType] if movingAvForBnOverXBatches > 0 else False # For the 1st layer, ask specific flag.
thisPathwayActivFuncPerLayer = [activationFunc] * thisPathwayNumOfLayers
thisPathwayActivFuncPerLayer[0] = "linear" if thisPathwayType != pt.FC else activationFunc # To not apply activation on raw input. -1 is linear activation.
thisPathway.makeLayersOfThisPathwayAndReturnDimensionsOfOutputFM(log,
rng,
inputToPathwayTrain,
inputToPathwayVal,
inputToPathwayTest,
inputToPathwayShapeTrain,
inputToPathwayShapeVal,
inputToPathwayShapeTest,
thisPathWayNKerns,
thisPathWayKernelDimensions,
convWInitMethod,
thisPathwayUseBnPerLayer,
movingAvForBnOverXBatches,
thisPathwayActivFuncPerLayer,
dropoutRatesForAllPathways[thisPathwayType],
maxPoolingParamsStructure[thisPathwayType],
indicesOfLowerRankLayersPerPathway[thisPathwayType],
ranksOfLowerRankLayersForEachPathway[thisPathwayType],
indicesOfLayersToConnectResidualsInOutput[thisPathwayType]
)
[dimsOfOutputFrom1stPathwayTrain, dimsOfOutputFrom1stPathwayVal, dimsOfOutputFrom1stPathwayTest] = thisPathway.getShapeOfOutput()
#=======================Make the SUBSAMPLED PATHWAYs of the CNN=============================
for subpath_i in range(self.numSubsPaths) :
thisPathway = SubsampledPathway(subsampleFactorsPerSubPath[subpath_i])
self.pathways.append(thisPathway) # There will be at least an entry as a secondary pathway. But it won't have any layers if it was not actually used.
thisPathwayType = thisPathway.pType()
inputToPathwayTrain = self._inp_x['train']['x_sub_'+str(subpath_i)]
inputToPathwayVal = self._inp_x['val']['x_sub_'+str(subpath_i)]
inputToPathwayTest = self._inp_x['test']['x_sub_'+str(subpath_i)]
thisPathWayNKerns = nkernsSubsampled[subpath_i]
thisPathWayKernelDimensions = kernelDimensionsSubsampled
thisPathwayNumOfLayers = len(thisPathWayNKerns)
thisPathwayUseBnPerLayer = [movingAvForBnOverXBatches > 0] * thisPathwayNumOfLayers
thisPathwayUseBnPerLayer[0] = applyBnToInputOfPathways[thisPathwayType] if movingAvForBnOverXBatches > 0 else False # For the 1st layer, ask specific flag.
thisPathwayActivFuncPerLayer = [activationFunc] * thisPathwayNumOfLayers
thisPathwayActivFuncPerLayer[0] = "linear" if thisPathwayType != pt.FC else activationFunc # To not apply activation on raw input. -1 is linear activation.
inputToPathwayShapeTrain = [None, numberOfImageChannelsPath2] + thisPathway.calcInputRczDimsToProduceOutputFmsOfCompatibleDims(thisPathWayKernelDimensions, dimsOfOutputFrom1stPathwayTrain);
inputToPathwayShapeVal = [None, numberOfImageChannelsPath2] + thisPathway.calcInputRczDimsToProduceOutputFmsOfCompatibleDims(thisPathWayKernelDimensions, dimsOfOutputFrom1stPathwayVal)
inputToPathwayShapeTest = [None, numberOfImageChannelsPath2] + thisPathway.calcInputRczDimsToProduceOutputFmsOfCompatibleDims(thisPathWayKernelDimensions, dimsOfOutputFrom1stPathwayTest)
thisPathway.makeLayersOfThisPathwayAndReturnDimensionsOfOutputFM(log,
rng,
inputToPathwayTrain,
inputToPathwayVal,
inputToPathwayTest,
inputToPathwayShapeTrain,
inputToPathwayShapeVal,
inputToPathwayShapeTest,
thisPathWayNKerns,
thisPathWayKernelDimensions,
convWInitMethod,
thisPathwayUseBnPerLayer,
movingAvForBnOverXBatches,
thisPathwayActivFuncPerLayer,
dropoutRatesForAllPathways[thisPathwayType],
maxPoolingParamsStructure[thisPathwayType],
indicesOfLowerRankLayersPerPathway[thisPathwayType],
ranksOfLowerRankLayersForEachPathway[thisPathwayType],
indicesOfLayersToConnectResidualsInOutput[thisPathwayType]
)
# this creates essentially the "upsampling layer"
thisPathway.upsampleOutputToNormalRes(upsamplingScheme="repeat",
shapeToMatchInRczTrain=dimsOfOutputFrom1stPathwayTrain,
shapeToMatchInRczVal=dimsOfOutputFrom1stPathwayVal,
shapeToMatchInRczTest=dimsOfOutputFrom1stPathwayTest)
#====================================CONCATENATE the output of the 2 cnn-pathways=============================
inputToFirstFcLayerTrain = None; inputToFirstFcLayerVal = None; inputToFirstFcLayerTest = None; numberOfFmsOfInputToFirstFcLayer = 0
for path_i in range(len(self.pathways)) :
[outputNormResOfPathTrain, outputNormResOfPathVal, outputNormResOfPathTest] = self.pathways[path_i].getOutputAtNormalRes()
[dimsOfOutputNormResOfPathTrain, dimsOfOutputNormResOfPathVal, dimsOfOutputNormResOfPathTest] = self.pathways[path_i].getShapeOfOutputAtNormalRes()
inputToFirstFcLayerTrain = tf.concat([inputToFirstFcLayerTrain, outputNormResOfPathTrain], axis=1) if path_i != 0 else outputNormResOfPathTrain
inputToFirstFcLayerVal = tf.concat([inputToFirstFcLayerVal, outputNormResOfPathVal], axis=1) if path_i != 0 else outputNormResOfPathVal
inputToFirstFcLayerTest = tf.concat([inputToFirstFcLayerTest, outputNormResOfPathTest], axis=1) if path_i != 0 else outputNormResOfPathTest
numberOfFmsOfInputToFirstFcLayer += dimsOfOutputNormResOfPathTrain[1]
#======================= Make the Fully Connected Layers =======================
thisPathway = FcPathway()
self.pathways.append(thisPathway)
thisPathwayType = thisPathway.pType()
# This is the shape of the kernel in the first FC layer.
# NOTE: If there is no hidden FC layer, this kernel is used in the Classification layer then.
# Originally it was 1x1x1 only. The pathways themselves where taking care of the receptive field.
# However I can now define it larger (eg 3x3x3), in case it helps combining the multiresolution features better/smoother.
# The convolution is seamless, ie same shape output/input, by mirror padding the input.
firstFcLayerAfterConcatenationKernelShape = self.kernelDimensionsFirstFcLayer
voxelsToPadPerDim = [ kernelDim - 1 for kernelDim in firstFcLayerAfterConcatenationKernelShape ]
log.print3("DEBUG: Shape of the kernel of the first FC layer is : " + str(firstFcLayerAfterConcatenationKernelShape))
log.print3("DEBUG: Input to the FC Pathway will be padded by that many voxels per dimension: " + str(voxelsToPadPerDim))
inputToPathwayTrain = padImageWithMirroring(inputToFirstFcLayerTrain, voxelsToPadPerDim)
inputToPathwayVal = padImageWithMirroring(inputToFirstFcLayerVal, voxelsToPadPerDim)
inputToPathwayTest = padImageWithMirroring(inputToFirstFcLayerTest, voxelsToPadPerDim)
inputToPathwayShapeTrain = [None, numberOfFmsOfInputToFirstFcLayer] + dimsOfOutputFrom1stPathwayTrain[2:5]
inputToPathwayShapeVal = [None, numberOfFmsOfInputToFirstFcLayer] + dimsOfOutputFrom1stPathwayVal[2:5]
inputToPathwayShapeTest = [None, numberOfFmsOfInputToFirstFcLayer] + dimsOfOutputFrom1stPathwayTest[2:5]
for rcz_i in range(3) :
inputToPathwayShapeTrain[2+rcz_i] += voxelsToPadPerDim[rcz_i]
inputToPathwayShapeVal[2+rcz_i] += voxelsToPadPerDim[rcz_i]
inputToPathwayShapeTest[2+rcz_i] += voxelsToPadPerDim[rcz_i]
thisPathWayNKerns = fcLayersFMs + [self.num_classes]
thisPathWayKernelDimensions = [firstFcLayerAfterConcatenationKernelShape] + [[1, 1, 1]] * (len(thisPathWayNKerns) - 1)
thisPathwayNumOfLayers = len(thisPathWayNKerns)
thisPathwayUseBnPerLayer = [movingAvForBnOverXBatches > 0] * thisPathwayNumOfLayers
thisPathwayUseBnPerLayer[0] = applyBnToInputOfPathways[thisPathwayType] if movingAvForBnOverXBatches > 0 else False # For the 1st layer, ask specific flag.
thisPathwayActivFuncPerLayer = [activationFunc] * thisPathwayNumOfLayers
thisPathwayActivFuncPerLayer[0] = "linear" if thisPathwayType != pt.FC else activationFunc # To not apply activation on raw input. -1 is linear activation.
thisPathway.makeLayersOfThisPathwayAndReturnDimensionsOfOutputFM(log,
rng,
inputToPathwayTrain,
inputToPathwayVal,
inputToPathwayTest,
inputToPathwayShapeTrain,
inputToPathwayShapeVal,
inputToPathwayShapeTest,
thisPathWayNKerns,
thisPathWayKernelDimensions,
convWInitMethod,
thisPathwayUseBnPerLayer,
movingAvForBnOverXBatches,
thisPathwayActivFuncPerLayer,
dropoutRatesForAllPathways[thisPathwayType],
maxPoolingParamsStructure[thisPathwayType],
indicesOfLowerRankLayersPerPathway[thisPathwayType],
ranksOfLowerRankLayersForEachPathway[thisPathwayType],
indicesOfLayersToConnectResidualsInOutput[thisPathwayType]
)
# =========== Make the final Target Layer (softmax, regression, whatever) ==========
log.print3("Adding the final Softmax Target layer...")
self.finalTargetLayer = self._getClassificationLayer()
self.finalTargetLayer.makeLayer(rng, self.getFcPathway().getLayer(-1), softmaxTemperature)
(self._output_gt_tensor_feeds['train']['y_gt'],
self._output_gt_tensor_feeds['val']['y_gt']) = self.finalTargetLayer.get_output_gt_tensor_feed()
log.print3("Finished building the CNN's model.")
def calcRecFieldOfPathway(self, kernelDimsPerLayer) :
return calcRecFieldFromKernDimListPerLayerWhenStrides1(kernelDimsPerLayer)