def __init__( self, cnnModelName, sessionLogger, mainOutputAbsFolder, folderForSessionCnnModels, #===MODEL PARAMETERS=== numberClasses, numberOfInputChannelsNormal, #===Normal pathway=== numFMsNormal, kernDimNormal, residConnAtLayersNormal, #==Subsampled pathway== useSubsampledBool, numFMsSubsampled, kernDimSubsampled, subsampleFactor, residConnAtLayersSubsampled, #==FC Layers==== numFMsFc, kernelDimensionsFirstFcLayer, residConnAtLayersFc, #==Size of Image Segments == segmDimTrain, segmDimVal, segmDimInfer, #== Batch Sizes == batchSizeTrain, batchSizeVal, batchSizeInfer, #==Dropout Rates== dropNormal, dropSubsampled, dropFc, #==Regularization== l1Reg, l2Reg, #== Weight Initialization== initialMethod, activationFunction, #== Batch Normalization == bnRollingAverOverThatManyBatches, #====Optimization===== learningRate, optimizerSgd0Adam1Rms2, classicMom0Nesterov1, momentumValue, momNonNormalized0Normalized1, #Adam b1Adam, b2Adam, eAdam, #Rms rhoRms, eRms ): #Importants for running session. self.cnnModelName = cnnModelName if cnnModelName else getDefaultModelName() self.sessionLogger = sessionLogger self.mainOutputAbsFolder = mainOutputAbsFolder self.pathAndFilenameToSaveModel = os.path.abspath(folderForSessionCnnModels + "/" + self.cnnModelName) #===========MODEL PARAMETERS========== self.numberClasses = numberClasses if numberClasses <> None else self.errReqNumClasses() self.numberOfInputChannelsNormal = numberOfInputChannelsNormal if numberOfInputChannelsNormal <> None or\ numberOfInputChannelsNormal<1 else self.errReqNumChannels() #===Normal pathway=== self.numFMsPerLayerNormal = numFMsNormal if numFMsNormal <> None and numFMsNormal > 0 else self.errReqFMsNormal() numOfLayers = len(self.numFMsPerLayerNormal) self.kernDimPerLayerNormal = kernDimNormal if checkKernDimPerLayerCorrect3dAndNumLayers(kernDimNormal, numOfLayers) else self.errReqKernDimNormal() self.receptiveFieldNormal = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(self.kernDimPerLayerNormal) residConnAtLayersNormal = residConnAtLayersNormal if residConnAtLayersNormal <> None else [] #layer number, starting from 1 for 1st layer. NOT indices. #==Subsampled pathway== self.useSubsampledBool = useSubsampledBool if useSubsampledBool <> None else False if not self.useSubsampledBool : self.numFMsPerLayerSubsampled = [] self.kernDimPerLayerSubsampled = [] self.receptiveFieldSubsampled = [] self.subsampleFactor = [] residConnAtLayersSubsampled = [] else : self.numFMsPerLayerSubsampled = numFMsSubsampled if numFMsSubsampled <> None else self.numFMsPerLayerNormal if kernDimSubsampled == None and\ len(self.numFMsPerLayerSubsampled) == len(self.numFMsPerLayerNormal) : self.kernDimPerLayerSubsampled = self.kernDimPerLayerNormal self.receptiveFieldSubsampled = self.receptiveFieldNormal elif kernDimSubsampled == None and\ len(self.numFMsPerLayerSubsampled) <> len(self.numFMsPerLayerNormal) : #user specified subsampled layers. self.errorRequireKernelDimensionsSubsampled(self.kernDimPerLayerNormal, numFMsSubsampled) # kernDimSubsampled was specified. Now it's going to be tricky to make sure everything alright. elif not checkKernDimPerLayerCorrect3dAndNumLayers(kernDimSubsampled, len(self.numFMsPerLayerSubsampled)) : self.errReqKernDimNormalCorr() else : #kernel dimensions specified and are correct (3d, same number of layers as subsampled specified). Need to check the two receptive fields and make sure they are correct. self.kernDimPerLayerSubsampled = kernDimSubsampled self.receptiveFieldSubsampled = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(self.kernDimPerLayerSubsampled) if self.receptiveFieldNormal <> self.receptiveFieldSubsampled : self.errorReceptiveFieldsOfNormalAndSubsampledDifferent(self.receptiveFieldNormal, self.receptiveFieldSubsampled) #Everything alright, finally. Proceed safely... self.subsampleFactor = subsampleFactor if subsampleFactor <> None else [3,3,3] if len(self.subsampleFactor) <> 3 : self.errorSubFactor3d() if not checkSubsampleFactorEven(self.subsampleFactor) : self.warnSubFactorOdd() residConnAtLayersSubsampled = residConnAtLayersSubsampled if residConnAtLayersSubsampled <> None else residConnAtLayersNormal #==FC Layers== self.numFMsInExtraFcs = numFMsFc if numFMsFc <> None else [] self.kernelDimensionsFirstFcLayer = kernelDimensionsFirstFcLayer if kernelDimensionsFirstFcLayer <> None else [1,1,1] assert len(self.kernelDimensionsFirstFcLayer) == 3 and (False not in [ dim > 0 for dim in self.kernelDimensionsFirstFcLayer] ) residConnAtLayersFc = residConnAtLayersFc if residConnAtLayersFc <> None else [] #==Size of Image Segments == self.segmDimNormalTrain = segmDimTrain if segmDimTrain <> None else self.errReqSegmDimTrain() self.segmDimNormalVal = segmDimVal if segmDimVal <> None else self.receptiveFieldNormal self.segmDimNormalInfer = segmDimInfer if segmDimInfer <> None else self.segmDimNormalTrain for (tr0_val1_inf2, segmentDimensions) in [ (0,self.segmDimNormalTrain), (1,self.segmDimNormalVal), (2,self.segmDimNormalInfer) ] : if not checkReceptiveFieldFineInComparisonToSegmentSize(self.receptiveFieldNormal, segmentDimensions) : self.errorSegmDimensionsSmallerThanReceptiveF(self.receptiveFieldNormal, segmentDimensions, tr0_val1_inf2) #=== Batch Sizes === self.batchSizeTrain = batchSizeTrain if batchSizeTrain <> None else self.errReqBatchSizeTr() self.batchSizeVal = batchSizeVal if batchSizeVal <> None else self.batchSizeTrain self.batchSizeInfer = batchSizeInfer if batchSizeInfer <> None else self.batchSizeTrain #=== Dropout rates === self.dropNormal = dropNormal if dropNormal <> None else [] self.dropSubsampled = dropSubsampled if dropSubsampled <> None else [] self.dropFc = dropFc if dropFc <> None else self.defaultDropFcList(self.numFMsInExtraFcs) #default = [0.0, 0.5, ..., 0.5] self.dropoutRatesForAllPathways = [self.dropNormal, self.dropSubsampled, self.dropFc, []] #==Regularization== self.l1Reg = l1Reg if l1Reg <> None else 0.000001 self.l2Reg = l2Reg if l2Reg <> None else 0.0001 #== Weight Initialization== self.initialMethodClassic0Delving1 = initialMethod if initialMethod <> None else 1 if not self.initialMethodClassic0Delving1 in [0,1]: self.errorReqInitializationMethod01() #== Activation Function == self.activationFunctionRelu0Prelu1 = activationFunction if activationFunction <> None else 1 if not self.activationFunctionRelu0Prelu1 in [0,1]: self.errorReqActivFunction01() #==BATCH NORMALIZATION== self.applyBnToInputOfPathways = [False, False, "Placeholder", False] # the 3 entry, for FC, is always True internally. self.bnRollingAverOverThatManyBatches = bnRollingAverOverThatManyBatches if bnRollingAverOverThatManyBatches <> None else 60 #====Optimization===== self.learningRate = learningRate if learningRate <> None else 0.001 self.optimizerSgd0Adam1Rms2 = optimizerSgd0Adam1Rms2 if optimizerSgd0Adam1Rms2 <> None else 2 if self.optimizerSgd0Adam1Rms2 == 0 : self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder"; self.rhoRms = "placeholder"; self.eRms = "placeholder"; elif self.optimizerSgd0Adam1Rms2 == 1 : self.b1Adam = b1Adam if b1Adam <> None else 0.9 #default in paper and seems good self.b2Adam = b2Adam if b2Adam <> None else 0.999 #default in paper and seems good self.eAdam = eAdam if eAdam else 10**(-8) self.rhoRms = "placeholder"; self.eRms = "placeholder"; elif self.optimizerSgd0Adam1Rms2 == 2 : self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder"; self.rhoRms = rhoRms if rhoRms <> None else 0.9 #default in paper and seems good self.eRms = eRms if eRms <> None else 10**(-4) # 1e-6 was the default in the paper, but blew up the gradients in first try. Never tried 1e-5 yet. else : self.errorRequireOptimizer012() self.classicMom0Nesterov1 = classicMom0Nesterov1 if classicMom0Nesterov1 <> None else 1 if self.classicMom0Nesterov1 not in [0,1]: self.errorRequireMomentumClass0Nestov1() self.momNonNormalized0Normalized1 = momNonNormalized0Normalized1 if momNonNormalized0Normalized1 <> None else 1 if self.momNonNormalized0Normalized1 not in [0,1] : self.errorRequireMomNonNorm0Norm1() self.momentumValue = momentumValue if momentumValue <> None else 0.6 if self.momentumValue < 0. or self.momentumValue > 1: self.errorRequireMomValueBetween01() #==============CALCULATED===================== if self.useSubsampledBool : self.segmDimSubsampledTrain = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalTrain, self.receptiveFieldNormal, self.subsampleFactor) self.segmDimSubsampledVal = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalVal, self.receptiveFieldNormal, self.subsampleFactor) self.segmDimSubsampledInfer = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalInfer, self.receptiveFieldNormal, self.subsampleFactor) else : self.segmDimSubsampledTrain = []; self.segmDimSubsampledVal = []; self.segmDimSubsampledInfer = []; # Residual Connections backwards, per pathway type : self.checkLayersForResidualsGivenDoNotInclude1st(residConnAtLayersNormal, residConnAtLayersSubsampled, residConnAtLayersFc) # The following variable passed to the system takes indices, ie number starts from 0. User specifies from 1. self.indicesOfLayersToConnectResidualsInOutput = [ [ layerNum - 1 for layerNum in residConnAtLayersNormal ], [ layerNum - 1 for layerNum in residConnAtLayersSubsampled ], [ layerNum - 1 for layerNum in residConnAtLayersFc ], [] ] #============= HIDDENS ====================== self.costFunctionLetter = "L" self.numberOfInputChannelsSubsampled = self.numberOfInputChannelsNormal #----for the zoomed-in pathway---- self.zoomedInPatchDimensions = [9, 9, 9] self.nkernsZoomedIn1 = [] #[15,15,15,30] self.kernelDimensionsZoomedIn1 = [[3,3,3], [3,3,3], [3,3,3], [3,3,3]] #MultiscaleConnections: self.convLayersToConnectToFirstFcForMultiscaleFromAllLayerTypes = [ [], [] ] #a sublist for each pathway. Starts from 0 index. Give a sublist, even empty for no connections. #... It's ok if I dont have a 2nd path but still give a 2nd sublist, it's controlled by nkernsSubsampled. #-------POOLING---------- (not fully supported currently) #One entry per pathway-type. leave [] if the pathway does not exist or there is no mp there AT ALL. #Inside each entry, put a list FOR EACH LAYER. It should be [] for the layer if no mp there. But FOR EACH LAYER. #MP is applied >>AT THE INPUT of the layer<<. To use mp to a layer, put a list of [[dsr,dsc,dsz], [strr,strc,strz], [mirrorPad-r,-c,-z], mode] which give the dimensions of the mp window, the stride, how many times to mirror the last slot at each dimension for padding (give 0 for none), the mode (usually 'max' pool). Eg [[2,2,2],[1,1,1]] or [[2,2,2],[2,2,2]] usually. self.maxPoolingParamsStructure = [ #If a pathway is not used, put an empty list in the first dimension entry. [ [] for layeri in xrange(len(self.numFMsPerLayerNormal)) ], #[[[2,2,2], [1,1,1], [1,1,1], 'max'], [],[],[],[],[],[], []], #first pathway [ [] for layeri in xrange(len(self.numFMsPerLayerSubsampled)) ], #second pathway [ [] for layeri in xrange(len(self.numFMsInExtraFcs) + 1) ], #FC. This should NEVER be used for segmentation. Possible for classification though. [[],[],[],[]] #zoomed in pathway. ] self.softmaxTemperature = 1.0 #Higher temperatures make the probabilities LESS distinctable. Actions have more similar probabilities.
def __init__( self, cnnModelName, sessionLogger, mainOutputAbsFolder, folderForSessionCnnModels, #===MODEL PARAMETERS=== numberClasses, numberOfInputChannelsNormal, #===Normal pathway=== numFMsNormal, kernDimNormal, #==Subsampled pathway== useSubsampledBool, numFMsSubsampled, kernDimSubsampled, subsampleFactor, #==FC Layers==== numFMsFc, #==Size of Image Segments == segmDimTrain, segmDimVal, segmDimInfer, #== Batch Sizes == batchSizeTrain, batchSizeVal, batchSizeInfer, #==Dropout Rates== dropNormal, dropSubsampled, dropFc, #==Regularization== l1Reg, l2Reg, #== Weight Initialization== initialMethod, activationFunction, #== Batch Normalization == bnRollingAverOverThatManyBatches, #====Optimization===== learningRate, optimizerSgd0Adam1Rms2, classicMom0Nesterov1, momentumValue, momNonNormalized0Normalized1, #Adam b1Adam, b2Adam, eAdam, #Rms rhoRms, eRms ): #Importants for running session. self.cnnModelName = cnnModelName if cnnModelName else getDefaultModelName() self.sessionLogger = sessionLogger self.mainOutputAbsFolder = mainOutputAbsFolder self.pathAndFilenameToSaveModel = os.path.abspath(folderForSessionCnnModels + "/" + self.cnnModelName) #===========MODEL PARAMETERS========== self.numberClasses = numberClasses if numberClasses <> None else self.errReqNumClasses() self.numberOfInputChannelsNormal = numberOfInputChannelsNormal if numberOfInputChannelsNormal <> None or\ numberOfInputChannelsNormal<1 else self.errReqNumChannels() #===Normal pathway=== self.numFMsPerLayerNormal = numFMsNormal if numFMsNormal <> None and numFMsNormal > 0 else self.errReqFMsNormal() numOfLayers = len(self.numFMsPerLayerNormal) self.kernDimPerLayerNormal = kernDimNormal if checkKernDimPerLayerCorrect3dAndNumLayers(kernDimNormal, numOfLayers) else self.errReqKernDimNormal() self.receptiveFieldNormal = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(self.kernDimPerLayerNormal) #==Subsampled pathway== self.useSubsampledBool = useSubsampledBool if useSubsampledBool <> None else False if not self.useSubsampledBool : self.numFMsPerLayerSubsampled = [] self.kernDimPerLayerSubsampled = [] self.receptiveFieldSubsampled = [] self.subsampleFactor = [] else : self.numFMsPerLayerSubsampled = numFMsSubsampled if numFMsSubsampled <> None else self.numFMsPerLayerNormal if kernDimSubsampled == None and\ len(self.numFMsPerLayerSubsampled) == len(self.numFMsPerLayerNormal) : self.kernDimPerLayerSubsampled = self.kernDimPerLayerNormal self.receptiveFieldSubsampled = self.receptiveFieldNormal elif kernDimSubsampled == None and\ len(self.numFMsPerLayerSubsampled) <> len(self.numFMsPerLayerNormal) : #user specified subsampled layers. self.errorRequireKernelDimensionsSubsampled(self.kernDimPerLayerNormal, numFMsSubsampled) # kernDimSubsampled was specified. Now it's going to be tricky to make sure everything alright. elif not checkKernDimPerLayerCorrect3dAndNumLayers(kernDimSubsampled, len(self.numFMsPerLayerSubsampled)) : self.errReqKernDimNormalCorr() else : #kernel dimensions specified and are correct (3d, same number of layers as subsampled specified). Need to check the two receptive fields and make sure they are correct. self.kernDimPerLayerSubsampled = kernDimSubsampled self.receptiveFieldSubsampled = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(self.kernDimPerLayerSubsampled) if self.receptiveFieldNormal <> self.receptiveFieldSubsampled : self.errorReceptiveFieldsOfNormalAndSubsampledDifferent(self.receptiveFieldNormal, self.receptiveFieldSubsampled) #Everything alright, finally. Proceed safely... self.subsampleFactor = subsampleFactor if subsampleFactor <> None else [3,3,3] if len(self.subsampleFactor) <> 3 : self.errorSubFactor3d() if not checkSubsampleFactorEven(self.subsampleFactor) : self.warnSubFactorOdd() #==FC Layers== self.numFMsInExtraFcs = numFMsFc if numFMsFc <> None else [] #==Size of Image Segments == self.segmDimNormalTrain = segmDimTrain if segmDimTrain <> None else self.errReqSegmDimTrain() self.segmDimNormalVal = segmDimVal if segmDimVal <> None else self.receptiveFieldNormal self.segmDimNormalInfer = segmDimInfer if segmDimInfer <> None else self.segmDimNormalTrain for (tr0_val1_inf2, segmentDimensions) in [ (0,self.segmDimNormalTrain), (1,self.segmDimNormalVal), (2,self.segmDimNormalInfer) ] : if not checkReceptiveFieldFineInComparisonToSegmentSize(self.receptiveFieldNormal, segmentDimensions) : self.errorSegmDimensionsSmallerThanReceptiveF(self.receptiveFieldNormal, segmentDimensions, tr0_val1_inf2) #=== Batch Sizes === self.batchSizeTrain = batchSizeTrain if batchSizeTrain <> None else self.errReqBatchSizeTr() self.batchSizeVal = batchSizeVal if batchSizeVal <> None else self.batchSizeTrain self.batchSizeInfer = batchSizeInfer if batchSizeInfer <> None else self.batchSizeTrain #=== Dropout rates === self.dropNormal = dropNormal if dropNormal <> None else [] self.dropSubsampled = dropSubsampled if dropSubsampled <> None else [] self.dropFc = dropFc if dropFc <> None else self.defaultDropFcList(self.numFMsInExtraFcs) #default = [0.0, 0.5, ..., 0.5] self.dropoutRatesForAllPathways = [self.dropNormal, self.dropSubsampled, self.dropFc, []] #==Regularization== self.l1Reg = l1Reg if l1Reg <> None else 0.000001 self.l2Reg = l2Reg if l2Reg <> None else 0.0001 #== Weight Initialization== self.initialMethodClassic0Delving1 = initialMethod if initialMethod <> None else 1 if not self.initialMethodClassic0Delving1 in [0,1]: self.errorReqInitializationMethod01() #== Activation Function == self.activationFunctionRelu0Prelu1 = activationFunction if activationFunction <> None else 1 if not self.activationFunctionRelu0Prelu1 in [0,1]: self.errorReqActivFunction01() #==BATCH NORMALIZATION== self.bnRollingAverOverThatManyBatches = bnRollingAverOverThatManyBatches if bnRollingAverOverThatManyBatches <> None else 60 #====Optimization===== self.learningRate = learningRate if learningRate <> None else 0.001 self.optimizerSgd0Adam1Rms2 = optimizerSgd0Adam1Rms2 if optimizerSgd0Adam1Rms2 <> None else 2 if self.optimizerSgd0Adam1Rms2 == 0 : self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder"; self.rhoRms = "placeholder"; self.eRms = "placeholder"; elif self.optimizerSgd0Adam1Rms2 == 1 : self.b1Adam = b1Adam if b1Adam <> None else 0.9 #default in paper and seems good self.b2Adam = b2Adam if b2Adam <> None else 0.999 #default in paper and seems good self.eAdam = eAdam if eAdam else 10**(-8) self.rhoRms = "placeholder"; self.eRms = "placeholder"; elif self.optimizerSgd0Adam1Rms2 == 2 : self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder"; self.rhoRms = rhoRms if rhoRms <> None else 0.9 #default in paper and seems good self.eRms = eRms if eRms <> None else 10**(-4) # 1e-6 was the default in the paper, but blew up the gradients in first try. Never tried 1e-5 yet. else : self.errorRequireOptimizer012() self.classicMom0Nesterov1 = classicMom0Nesterov1 if classicMom0Nesterov1 <> None else 1 if self.classicMom0Nesterov1 not in [0,1]: self.errorRequireMomentumClass0Nestov1() self.momNonNormalized0Normalized1 = momNonNormalized0Normalized1 if momNonNormalized0Normalized1 <> None else 1 if self.momNonNormalized0Normalized1 not in [0,1] : self.errorRequireMomNonNorm0Norm1() self.momentumValue = momentumValue if momentumValue <> None else 0.6 if self.momentumValue < 0. or self.momentumValue > 1: self.errorRequireMomValueBetween01() #==============CALCULATED============================ if self.useSubsampledBool : self.segmDimSubsampledTrain = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalTrain, self.receptiveFieldNormal, self.subsampleFactor) self.segmDimSubsampledVal = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalVal, self.receptiveFieldNormal, self.subsampleFactor) self.segmDimSubsampledInfer = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalInfer, self.receptiveFieldNormal, self.subsampleFactor) else : self.segmDimSubsampledTrain = []; self.segmDimSubsampledVal = []; self.segmDimSubsampledInfer = []; #============= HIDDENS ====================== self.costFunctionLetter = "L" self.kernelDimensionsFirstFcLayer = [1,1,1] self.numberOfInputChannelsSubsampled = self.numberOfInputChannelsNormal #----for the zoomed-in pathway---- self.zoomedInPatchDimensions = [9, 9, 9] self.nkernsZoomedIn1 = [] #[15,15,15,30] self.kernelDimensionsZoomedIn1 = [[3,3,3], [3,3,3], [3,3,3], [3,3,3]] #MultiscaleConnections: self.convLayersToConnectToFirstFcForMultiscaleFromAllLayerTypes = [ [], [] ] #a sublist for each pathway. Starts from 0 index. Give a sublist, even empty for no connections. #... It's ok if I dont have a 2nd path but still give a 2nd sublist, it's controlled by nkernsSubsampled. #-------MAX POOLING---------- #One entry per pathway-type. leave [] if the pathway does not exist or there is no mp there AT ALL. #Inside each entry, put a list FOR EACH LAYER. It should be [] for the layer if no mp there. But FOR EACH LAYER. #MP is applied >>AT THE INPUT of the layer<<. To use mp to a layer, put a list of [[dsr,dsc,dsz], [strr,strc,strz], [mirrorPad-r,-c,-z], mode] which give the dimensions of the mp window, the stride, how many times to mirror the last slot at each dimension for padding (give 0 for none), the mode (usually 'max' pool). Eg [[2,2,2],[1,1,1]] or [[2,2,2],[2,2,2]] usually. self.maxPoolingParamsStructure = [ #If a pathway is not used, put an empty list in the first dimension entry. [ [] for layeri in xrange(len(self.numFMsPerLayerNormal)) ], #[[[2,2,2], [1,1,1], [1,1,1], 'max'], [],[],[],[],[],[], []], #first pathway [ [] for layeri in xrange(len(self.numFMsPerLayerSubsampled)) ], #second pathway [ [] for layeri in xrange(len(self.numFMsInExtraFcs)) ], #FC. I this this should NEVER be used. [[],[],[], []] #zoomed in pathway. ] self.softmaxTemperature = 1.0 #Higher temperatures make the probabilities LESS distinctable. Actions have more similar probabilities.
def calcRecFieldOfPathway(self, kernelDimsPerLayer) :
return calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(kernelDimsPerLayer)
def calcRecFieldOfPathway(self, kernelDimsPerLayer):
return calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(
kernelDimsPerLayer)
def __init__(
self,
cnnModelName,
sessionLogger,
mainOutputAbsFolder,
folderForSessionCnnModels,
#===MODEL PARAMETERS===
numberClasses,
numberOfInputChannelsNormal,
#===Normal pathway===
numFMsNormal,
kernDimNormal,
residConnAtLayersNormal,
lowerRankLayersNormal,
#==Subsampled pathway==
useSubsampledBool,
numFMsSubsampled,
kernDimSubsampled,
subsampleFactor,
residConnAtLayersSubsampled,
lowerRankLayersSubsampled,
#==FC Layers====
numFMsFc,
kernelDimensionsFirstFcLayer,
residConnAtLayersFc,
#==Size of Image Segments ==
segmDimTrain,
segmDimVal,
segmDimInfer,
#== Batch Sizes ==
batchSizeTrain,
batchSizeVal,
batchSizeInfer,
#===Other Architectural Parameters ===
activationFunction,
#==Dropout Rates==
dropNormal,
dropSubsampled,
dropFc,
#== Weight Initialization==
initialMethod,
#== Batch Normalization ==
bnRollingAverOverThatManyBatches):
#Importants for running session.
self.cnnModelName = cnnModelName if cnnModelName else getDefaultModelName(
)
self.sessionLogger = sessionLogger
self.mainOutputAbsFolder = mainOutputAbsFolder
self.pathAndFilenameToSaveModel = os.path.abspath(
folderForSessionCnnModels + "/" + self.cnnModelName)
#===========MODEL PARAMETERS==========
self.numberClasses = numberClasses if numberClasses <> None else self.errReqNumClasses(
)
self.numberOfInputChannelsNormal = numberOfInputChannelsNormal if numberOfInputChannelsNormal <> None or\
numberOfInputChannelsNormal<1 else self.errReqNumChannels()
#===Normal pathway===
self.numFMsPerLayerNormal = numFMsNormal if numFMsNormal <> None and numFMsNormal > 0 else self.errReqFMsNormal(
)
numOfLayers = len(self.numFMsPerLayerNormal)
self.kernDimPerLayerNormal = kernDimNormal if checkKernDimPerLayerCorrect3dAndNumLayers(
kernDimNormal, numOfLayers) else self.errReqKernDimNormal()
self.receptiveFieldNormal = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(
self.kernDimPerLayerNormal) # Just for COMPATIBILITY CHECKS!
residConnAtLayersNormal = residConnAtLayersNormal if residConnAtLayersNormal <> None else [
] #layer number, starting from 1 for 1st layer. NOT indices.
lowerRankLayersNormal = lowerRankLayersNormal if lowerRankLayersNormal <> None else [
] #layer number, starting from 1 for 1st layer. NOT indices.
#==Subsampled pathway==
self.useSubsampledBool = useSubsampledBool if useSubsampledBool <> None else False
if not self.useSubsampledBool:
self.numFMsPerLayerSubsampled = []
self.kernDimPerLayerSubsampled = []
self.receptiveFieldSubsampled = []
self.subsampleFactor = []
residConnAtLayersSubsampled = []
lowerRankLayersSubsampled = []
else:
self.numFMsPerLayerSubsampled = numFMsSubsampled if numFMsSubsampled <> None else self.numFMsPerLayerNormal
self.numFMsPerLayerSubsampled = self.changeDatastructureToListOfListsForSecondaryPathwaysIfNeeded(
self.numFMsPerLayerSubsampled)
# check that all subsampled pathways have the same number of layers. Limitation in the code currently, because I use kernDimSubsampled for all of them.
if not self.checkThatSublistsHaveSameLength(
self.numFMsPerLayerSubsampled):
self.errReqSameNumOfLayersPerSubPathway()
numOfLayersInEachSubPath = len(self.numFMsPerLayerSubsampled[0])
if kernDimSubsampled == None and numOfLayersInEachSubPath == len(
self.numFMsPerLayerNormal):
self.kernDimPerLayerSubsampled = self.kernDimPerLayerNormal
self.receptiveFieldSubsampled = self.receptiveFieldNormal
elif kernDimSubsampled == None and numOfLayersInEachSubPath <> len(
self.numFMsPerLayerNormal
): #user specified subsampled layers.
self.errorRequireKernelDimensionsSubsampled(
self.kernDimPerLayerNormal, numFMsSubsampled)
# kernDimSubsampled was specified. Now it's going to be tricky to make sure everything alright.
elif not checkKernDimPerLayerCorrect3dAndNumLayers(
kernDimSubsampled, numOfLayersInEachSubPath):
self.errReqKernDimNormalCorr()
else: #kernel dimensions specified and are correct (3d, same number of layers as subsampled specified). Need to check the two receptive fields and make sure they are correct.
self.kernDimPerLayerSubsampled = kernDimSubsampled
self.receptiveFieldSubsampled = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(
self.kernDimPerLayerSubsampled)
if self.receptiveFieldNormal <> self.receptiveFieldSubsampled:
self.errorReceptiveFieldsOfNormalAndSubsampledDifferent(
self.receptiveFieldNormal,
self.receptiveFieldSubsampled)
#Everything alright, finally. Proceed safely...
self.subsampleFactor = subsampleFactor if subsampleFactor <> None else [
3, 3, 3
]
self.subsampleFactor = self.changeDatastructureToListOfListsForSecondaryPathwaysIfNeeded(
self.subsampleFactor)
for secondaryPathway_i in xrange(
len(self.subsampleFactor)
): #It should now be a list of lists, one sublist per secondary pathway. This is what is currently defining how many pathways to use.
if len(self.subsampleFactor[secondaryPathway_i]) <> 3:
self.errorSubFactor3d()
if not checkSubsampleFactorEven(
self.subsampleFactor[secondaryPathway_i]):
self.warnSubFactorOdd()
#---For multiple lower-scale pathways, via the numFMsPerLayerSubsampled and subsampleFactor config ----
numOfSubsPaths = max(len(self.numFMsPerLayerSubsampled),
len(self.subsampleFactor))
# Default behaviour:
# If less sublists in numFMsPerLayerSubsampled were given than numOfSubsPaths, add more sublists of subsampleFactors, for the extra subpaths.
for _ in range(numOfSubsPaths -
len(self.numFMsPerLayerSubsampled)):
numFmsForLayersOfLastSubPath = self.numFMsPerLayerSubsampled[
-1]
self.numFMsPerLayerSubsampled.append([
max(1, int(numFmsInLayer_i / 2))
for numFmsInLayer_i in numFmsForLayersOfLastSubPath
]) # half of how many previous subPath had.
# If less sublists in subsampleFactor were given than numOfSubsPaths, add more sublists of subsampleFactors, for the extra subpaths.
for _ in range(numOfSubsPaths - len(self.subsampleFactor)):
self.subsampleFactor.append(
[
subFactorInDim_i + 2
for subFactorInDim_i in self.subsampleFactor[-1]
]
) # Adds one more sublist, eg [5,5,5], which is the last subFactor, increased by +2 in all rcz dimensions.
# Residuals and lower ranks.
residConnAtLayersSubsampled = residConnAtLayersSubsampled if residConnAtLayersSubsampled <> None else residConnAtLayersNormal
lowerRankLayersSubsampled = lowerRankLayersSubsampled if lowerRankLayersSubsampled <> None else lowerRankLayersNormal
#==FC Layers==
self.numFMsInExtraFcs = numFMsFc if numFMsFc <> None else []
self.kernelDimensionsFirstFcLayer = kernelDimensionsFirstFcLayer if kernelDimensionsFirstFcLayer <> None else [
1, 1, 1
]
assert len(self.kernelDimensionsFirstFcLayer) == 3 and (False not in [
dim > 0 for dim in self.kernelDimensionsFirstFcLayer
])
residConnAtLayersFc = residConnAtLayersFc if residConnAtLayersFc <> None else []
#==Size of Image Segments ==
self.segmDimNormalTrain = segmDimTrain if segmDimTrain <> None else self.errReqSegmDimTrain(
)
self.segmDimNormalVal = segmDimVal if segmDimVal <> None else self.receptiveFieldNormal
self.segmDimNormalInfer = segmDimInfer if segmDimInfer <> None else self.segmDimNormalTrain
for (tr0_val1_inf2,
segmentDimensions) in [(0, self.segmDimNormalTrain),
(1, self.segmDimNormalVal),
(2, self.segmDimNormalInfer)]:
if not checkReceptiveFieldFineInComparisonToSegmentSize(
self.receptiveFieldNormal, segmentDimensions):
self.errorSegmDimensionsSmallerThanReceptiveF(
self.receptiveFieldNormal, segmentDimensions,
tr0_val1_inf2)
#=== Batch Sizes ===
self.batchSizeTrain = batchSizeTrain if batchSizeTrain <> None else self.errReqBatchSizeTr(
)
self.batchSizeVal = batchSizeVal if batchSizeVal <> None else self.batchSizeTrain
self.batchSizeInfer = batchSizeInfer if batchSizeInfer <> None else self.batchSizeTrain
#=== Dropout rates ===
self.dropNormal = dropNormal if dropNormal <> None else []
self.dropSubsampled = dropSubsampled if dropSubsampled <> None else []
self.dropFc = dropFc if dropFc <> None else self.defaultDropFcList(
self.numFMsInExtraFcs) #default = [0.0, 0.5, ..., 0.5]
self.dropoutRatesForAllPathways = [
self.dropNormal, self.dropSubsampled, self.dropFc, []
]
#== Weight Initialization==
self.initialMethodClassic0Delving1 = initialMethod if initialMethod <> None else 1
if not self.initialMethodClassic0Delving1 in [0, 1]:
self.errorReqInitializationMethod01()
#== Activation Function ==
self.activationFunctionRelu0Prelu1 = activationFunction if activationFunction <> None else 1
if not self.activationFunctionRelu0Prelu1 in [0, 1]:
self.errorReqActivFunction01()
#==BATCH NORMALIZATION==
self.applyBnToInputOfPathways = [
False, False, True, False
] # the 3 entry, for FC, should always be True.
self.bnRollingAverOverThatManyBatches = bnRollingAverOverThatManyBatches if bnRollingAverOverThatManyBatches <> None else 60
#==============CALCULATED=====================
# Residual Connections backwards, per pathway type :
self.checkLayersForResidualsGivenDoNotInclude1st(
residConnAtLayersNormal, residConnAtLayersSubsampled,
residConnAtLayersFc)
# The following variable passed to the system takes indices, ie number starts from 0. User specifies from 1.
self.indicesOfLayersToConnectResidualsInOutput = [
[layerNum - 1 for layerNum in residConnAtLayersNormal],
[layerNum - 1 for layerNum in residConnAtLayersSubsampled],
[layerNum - 1 for layerNum in residConnAtLayersFc], []
]
self.indicesOfLowerRankLayersPerPathway = [
[layerNum - 1 for layerNum in lowerRankLayersNormal],
[layerNum - 1 for layerNum in lowerRankLayersSubsampled],
[], #FC doesn't make sense to be lower rank. It's 1x1x1 anyway.
[]
]
self.ranksOfLowerRankLayersForEachPathway = [[
2 for layer_i in self.indicesOfLowerRankLayersPerPathway[0]
], [2 for layer_i in self.indicesOfLowerRankLayersPerPathway[1]], [],
[]]
#============= HIDDENS ======================
self.numberOfInputChannelsSubsampled = self.numberOfInputChannelsNormal
#MultiscaleConnections:
self.convLayersToConnectToFirstFcForMultiscaleFromAllLayerTypes = [
[], []
] #a sublist for each pathway. Starts from 0 index. Give a sublist, even empty for no connections.
#... It's ok if I dont have a 2nd path but still give a 2nd sublist, it's controlled by nkernsSubsampled.
#-------POOLING---------- (not fully supported currently)
#One entry per pathway-type. leave [] if the pathway does not exist or there is no mp there AT ALL.
#Inside each entry, put a list FOR EACH LAYER. It should be [] for the layer if no mp there. But FOR EACH LAYER.
#MP is applied >>AT THE INPUT of the layer<<. To use mp to a layer, put a list of [[dsr,dsc,dsz], [strr,strc,strz], [mirrorPad-r,-c,-z], mode] which give the dimensions of the mp window, the stride, how many times to mirror the last slot at each dimension for padding (give 0 for none), the mode (usually 'max' pool). Eg [[2,2,2],[1,1,1]] or [[2,2,2],[2,2,2]] usually.
#If a pathway is not used (eg subsampled), put an empty list in the first dimension entry.
mpParamsNorm = [
[] for layeri in xrange(len(self.numFMsPerLayerNormal))
] #[[[2,2,2], [1,1,1], [1,1,1], 'max'], [],[],[],[],[],[], []], #first pathway
mpParamsSubs = [
[] for layeri in xrange(len(self.numFMsPerLayerSubsampled[0]))
] if self.useSubsampledBool else [
] # CAREFUL about the [0]. Only here till this structure is made different per pathway and not pathwayType.
mpParamsFc = [
[] for layeri in xrange(len(self.numFMsInExtraFcs) + 1)
] #FC. This should NEVER be used for segmentation. Possible for classification though.
self.maxPoolingParamsStructure = [
mpParamsNorm, mpParamsSubs, mpParamsFc
]
self.softmaxTemperature = 1.0 #Higher temperatures make the probabilities LESS distinctable. Actions have more similar probabilities.