def __init__(	self,
			cnnModelName,
			sessionLogger,
			mainOutputAbsFolder,
			folderForSessionCnnModels,
			#===MODEL PARAMETERS===
			numberClasses,
			numberOfInputChannelsNormal,
			#===Normal pathway===
			numFMsNormal,
			kernDimNormal,
			#==Subsampled pathway==
			useSubsampledBool,
			numFMsSubsampled,
			kernDimSubsampled,
			subsampleFactor,
			#==FC Layers====
			numFMsFc,
			#==Size of Image Segments ==
			segmDimTrain,
			segmDimVal,
			segmDimInfer,
			#== Batch Sizes ==
			batchSizeTrain,
			batchSizeVal,
			batchSizeInfer,
			#==Dropout Rates==
			dropNormal,
			dropSubsampled,
			dropFc,
			#==Regularization==
			l1Reg,
			l2Reg,
			#== Weight Initialization==
			initialMethod,
			activationFunction,
			#== Batch Normalization ==
			bnRollingAverOverThatManyBatches,

			#====Optimization=====
			learningRate,
			optimizerSgd0Adam1Rms2,
			classicMom0Nesterov1,
			momentumValue,
			momNonNormalized0Normalized1,
			#Adam
			b1Adam,
			b2Adam,
			eAdam,
			#Rms
			rhoRms,
			eRms
			):

		#Importants for running session.

		self.cnnModelName = cnnModelName if cnnModelName else getDefaultModelName()
		self.sessionLogger = sessionLogger
		self.mainOutputAbsFolder = mainOutputAbsFolder
		self.pathAndFilenameToSaveModel = os.path.abspath(folderForSessionCnnModels + "/" + self.cnnModelName)

		#===========MODEL PARAMETERS==========
		self.numberClasses = numberClasses if numberClasses <> None else self.errReqNumClasses()
		self.numberOfInputChannelsNormal = numberOfInputChannelsNormal if numberOfInputChannelsNormal <> None or\
								numberOfInputChannelsNormal<1 else self.errReqNumChannels()

		#===Normal pathway===
		self.numFMsPerLayerNormal = numFMsNormal if numFMsNormal <> None and numFMsNormal > 0 else self.errReqFMsNormal()
		numOfLayers = len(self.numFMsPerLayerNormal)
		
		self.kernDimPerLayerNormal = kernDimNormal if checkKernDimPerLayerCorrect3dAndNumLayers(kernDimNormal, numOfLayers) else self.errReqKernDimNormal()
		self.receptiveFieldNormal = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(self.kernDimPerLayerNormal)
		#==Subsampled pathway==
		self.useSubsampledBool = useSubsampledBool if useSubsampledBool <> None else False
		if not self.useSubsampledBool :
			self.numFMsPerLayerSubsampled = []
			self.kernDimPerLayerSubsampled = []
			self.receptiveFieldSubsampled = []
			self.subsampleFactor = []

		else :
			self.numFMsPerLayerSubsampled = numFMsSubsampled if numFMsSubsampled <> None else self.numFMsPerLayerNormal
			if kernDimSubsampled == None and\
					len(self.numFMsPerLayerSubsampled) == len(self.numFMsPerLayerNormal) :
				self.kernDimPerLayerSubsampled = self.kernDimPerLayerNormal
				self.receptiveFieldSubsampled = self.receptiveFieldNormal
			elif kernDimSubsampled == None and\
					len(self.numFMsPerLayerSubsampled) <> len(self.numFMsPerLayerNormal) : #user specified subsampled layers.
				self.errorRequireKernelDimensionsSubsampled(self.kernDimPerLayerNormal, numFMsSubsampled)
			# kernDimSubsampled was specified. Now it's going to be tricky to make sure everything alright.
			elif not checkKernDimPerLayerCorrect3dAndNumLayers(kernDimSubsampled, len(self.numFMsPerLayerSubsampled)) :
				self.errReqKernDimNormalCorr()
			else : #kernel dimensions specified and are correct (3d, same number of layers as subsampled specified). Need to check the two receptive fields and make sure they are correct.
				self.kernDimPerLayerSubsampled = kernDimSubsampled
				self.receptiveFieldSubsampled = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(self.kernDimPerLayerSubsampled)
				if self.receptiveFieldNormal <> self.receptiveFieldSubsampled :
					self.errorReceptiveFieldsOfNormalAndSubsampledDifferent(self.receptiveFieldNormal, self.receptiveFieldSubsampled)
				#Everything alright, finally. Proceed safely...
			self.subsampleFactor = subsampleFactor if subsampleFactor <> None else [3,3,3]
			if len(self.subsampleFactor) <> 3 :
				self.errorSubFactor3d()
			if not checkSubsampleFactorEven(self.subsampleFactor) :
				self.warnSubFactorOdd()
		#==FC Layers==
		self.numFMsInExtraFcs = numFMsFc if numFMsFc <> None else []

		#==Size of Image Segments ==
		self.segmDimNormalTrain = segmDimTrain if segmDimTrain <> None else self.errReqSegmDimTrain()
		self.segmDimNormalVal = segmDimVal if segmDimVal <> None else self.receptiveFieldNormal
		self.segmDimNormalInfer = segmDimInfer if segmDimInfer <> None else self.segmDimNormalTrain
		for (tr0_val1_inf2, segmentDimensions) in [ (0,self.segmDimNormalTrain), (1,self.segmDimNormalVal), (2,self.segmDimNormalInfer) ] :
			if not checkReceptiveFieldFineInComparisonToSegmentSize(self.receptiveFieldNormal, segmentDimensions) :
				self.errorSegmDimensionsSmallerThanReceptiveF(self.receptiveFieldNormal, segmentDimensions, tr0_val1_inf2)


		#=== Batch Sizes ===
		self.batchSizeTrain = batchSizeTrain if batchSizeTrain <> None else self.errReqBatchSizeTr()
		self.batchSizeVal = batchSizeVal if batchSizeVal <> None else self.batchSizeTrain
		self.batchSizeInfer = batchSizeInfer if batchSizeInfer <> None else self.batchSizeTrain

		#=== Dropout rates ===
		self.dropNormal = dropNormal if dropNormal <> None else []
		self.dropSubsampled = dropSubsampled if dropSubsampled <> None else []
		self.dropFc = dropFc if dropFc <> None else self.defaultDropFcList(self.numFMsInExtraFcs) #default = [0.0, 0.5, ..., 0.5]
		self.dropoutRatesForAllPathways = [self.dropNormal, self.dropSubsampled, self.dropFc, []]

		#==Regularization==
		self.l1Reg = l1Reg if l1Reg <> None else 0.000001
		self.l2Reg = l2Reg if l2Reg <> None else 0.0001

		#== Weight Initialization==
		self.initialMethodClassic0Delving1 = initialMethod if initialMethod <> None else 1
		if not self.initialMethodClassic0Delving1 in [0,1]:
			self.errorReqInitializationMethod01()
		#== Activation Function ==
		self.activationFunctionRelu0Prelu1 = activationFunction if activationFunction <> None else 1
		if not self.activationFunctionRelu0Prelu1 in [0,1]:
			self.errorReqActivFunction01()

		#==BATCH NORMALIZATION==
		self.bnRollingAverOverThatManyBatches = bnRollingAverOverThatManyBatches if bnRollingAverOverThatManyBatches <> None else 60
		
		#====Optimization=====
		self.learningRate = learningRate if learningRate <> None else 0.001
		self.optimizerSgd0Adam1Rms2 = optimizerSgd0Adam1Rms2 if optimizerSgd0Adam1Rms2 <> None else 2
		if self.optimizerSgd0Adam1Rms2 == 0 :
			self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder";
			self.rhoRms = "placeholder"; self.eRms = "placeholder";
		elif self.optimizerSgd0Adam1Rms2 == 1 :
			self.b1Adam = b1Adam if b1Adam <> None else 0.9 #default in paper and seems good
			self.b2Adam = b2Adam if b2Adam <> None else 0.999 #default in paper and seems good
			self.eAdam = eAdam if eAdam else 10**(-8)
			self.rhoRms = "placeholder"; self.eRms = "placeholder";
		elif self.optimizerSgd0Adam1Rms2 == 2 :
			self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder";
			self.rhoRms = rhoRms if rhoRms <> None else 0.9 #default in paper and seems good
			self.eRms = eRms if eRms <> None else 10**(-4) # 1e-6 was the default in the paper, but blew up the gradients in first try. Never tried 1e-5 yet.
		else :
			self.errorRequireOptimizer012()

		self.classicMom0Nesterov1 = classicMom0Nesterov1 if classicMom0Nesterov1 <> None else 1
		if self.classicMom0Nesterov1 not in [0,1]:
			self.errorRequireMomentumClass0Nestov1()
		self.momNonNormalized0Normalized1 = momNonNormalized0Normalized1 if momNonNormalized0Normalized1 <> None else 1
		if self.momNonNormalized0Normalized1 not in [0,1] :
			self.errorRequireMomNonNorm0Norm1()
		self.momentumValue = momentumValue if momentumValue <> None else 0.6
		if self.momentumValue < 0. or self.momentumValue > 1:
			self.errorRequireMomValueBetween01()



		#==============CALCULATED============================
		if self.useSubsampledBool :
			self.segmDimSubsampledTrain = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalTrain, self.receptiveFieldNormal, self.subsampleFactor)
			self.segmDimSubsampledVal = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalVal, self.receptiveFieldNormal, self.subsampleFactor)
			self.segmDimSubsampledInfer = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalInfer, self.receptiveFieldNormal, self.subsampleFactor)
		else :
			self.segmDimSubsampledTrain = []; self.segmDimSubsampledVal = []; self.segmDimSubsampledInfer = [];
		#============= HIDDENS ======================

		self.costFunctionLetter = "L"
		self.kernelDimensionsFirstFcLayer = [1,1,1]

		self.numberOfInputChannelsSubsampled = self.numberOfInputChannelsNormal

		#----for the zoomed-in pathway----
		self.zoomedInPatchDimensions = [9, 9, 9]
		self.nkernsZoomedIn1 = [] #[15,15,15,30]
		self.kernelDimensionsZoomedIn1 = [[3,3,3], [3,3,3], [3,3,3], [3,3,3]]

		#MultiscaleConnections:
		self.convLayersToConnectToFirstFcForMultiscaleFromAllLayerTypes = [ [], [] ] #a sublist for each pathway. Starts from 0 index. Give a sublist, even empty for no connections.
		#... It's ok if I dont have a 2nd path but still give a 2nd sublist, it's controlled by nkernsSubsampled.

		#-------MAX POOLING----------
		#One entry per pathway-type. leave [] if the pathway does not exist or there is no mp there AT ALL.
		#Inside each entry, put a list FOR EACH LAYER. It should be [] for the layer if no mp there. But FOR EACH LAYER.
		#MP is applied >>AT THE INPUT of the layer<<. To use mp to a layer, put a list of [[dsr,dsc,dsz], [strr,strc,strz], [mirrorPad-r,-c,-z], mode] which give the dimensions of the mp window, the stride, how many times to mirror the last slot at each dimension for padding (give 0 for none), the mode (usually 'max' pool). Eg [[2,2,2],[1,1,1]] or [[2,2,2],[2,2,2]] usually.
		self.maxPoolingParamsStructure = [ #If a pathway is not used, put an empty list in the first dimension entry. 
						[ [] for layeri in xrange(len(self.numFMsPerLayerNormal)) ], #[[[2,2,2], [1,1,1], [1,1,1], 'max'], [],[],[],[],[],[], []], #first pathway
						[ [] for layeri in xrange(len(self.numFMsPerLayerSubsampled)) ], #second pathway
						[ [] for layeri in xrange(len(self.numFMsInExtraFcs)) ], #FC. I this this should NEVER be used.
						[[],[],[], []] #zoomed in pathway.
						]

		self.softmaxTemperature = 1.0 #Higher temperatures make the probabilities LESS distinctable. Actions have more similar probabilities. 
Exemple #2
0
	def __init__(	self,
			cnnModelName,
			sessionLogger,
			mainOutputAbsFolder,
			folderForSessionCnnModels,
			#===MODEL PARAMETERS===
			numberClasses,
			numberOfInputChannelsNormal,
			#===Normal pathway===
			numFMsNormal,
			kernDimNormal,
			residConnAtLayersNormal,
			#==Subsampled pathway==
			useSubsampledBool,
			numFMsSubsampled,
			kernDimSubsampled,
			subsampleFactor,
			residConnAtLayersSubsampled,
			#==FC Layers====
			numFMsFc,
			kernelDimensionsFirstFcLayer,
			residConnAtLayersFc,
			
			#==Size of Image Segments ==
			segmDimTrain,
			segmDimVal,
			segmDimInfer,
			#== Batch Sizes ==
			batchSizeTrain,
			batchSizeVal,
			batchSizeInfer,
			#==Dropout Rates==
			dropNormal,
			dropSubsampled,
			dropFc,
			#==Regularization==
			l1Reg,
			l2Reg,
			#== Weight Initialization==
			initialMethod,
			activationFunction,
			#== Batch Normalization ==
			bnRollingAverOverThatManyBatches,

			#====Optimization=====
			learningRate,
			optimizerSgd0Adam1Rms2,
			classicMom0Nesterov1,
			momentumValue,
			momNonNormalized0Normalized1,
			#Adam
			b1Adam,
			b2Adam,
			eAdam,
			#Rms
			rhoRms,
			eRms
			):

		#Importants for running session.

		self.cnnModelName = cnnModelName if cnnModelName else getDefaultModelName()
		self.sessionLogger = sessionLogger
		self.mainOutputAbsFolder = mainOutputAbsFolder
		self.pathAndFilenameToSaveModel = os.path.abspath(folderForSessionCnnModels + "/" + self.cnnModelName)

		#===========MODEL PARAMETERS==========
		self.numberClasses = numberClasses if numberClasses <> None else self.errReqNumClasses()
		self.numberOfInputChannelsNormal = numberOfInputChannelsNormal if numberOfInputChannelsNormal <> None or\
								numberOfInputChannelsNormal<1 else self.errReqNumChannels()

		#===Normal pathway===
		self.numFMsPerLayerNormal = numFMsNormal if numFMsNormal <> None and numFMsNormal > 0 else self.errReqFMsNormal()
		numOfLayers = len(self.numFMsPerLayerNormal)
		self.kernDimPerLayerNormal = kernDimNormal if checkKernDimPerLayerCorrect3dAndNumLayers(kernDimNormal, numOfLayers) else self.errReqKernDimNormal()
		self.receptiveFieldNormal = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(self.kernDimPerLayerNormal)
		residConnAtLayersNormal = residConnAtLayersNormal if residConnAtLayersNormal <> None else [] #layer number, starting from 1 for 1st layer. NOT indices.
		#==Subsampled pathway==
		self.useSubsampledBool = useSubsampledBool if useSubsampledBool <> None else False
		if not self.useSubsampledBool :
			self.numFMsPerLayerSubsampled = []
			self.kernDimPerLayerSubsampled = []
			self.receptiveFieldSubsampled = []
			self.subsampleFactor = []
			residConnAtLayersSubsampled = []
		
		else :
			self.numFMsPerLayerSubsampled = numFMsSubsampled if numFMsSubsampled <> None else self.numFMsPerLayerNormal
			if kernDimSubsampled == None and\
					len(self.numFMsPerLayerSubsampled) == len(self.numFMsPerLayerNormal) :
				self.kernDimPerLayerSubsampled = self.kernDimPerLayerNormal
				self.receptiveFieldSubsampled = self.receptiveFieldNormal
			elif kernDimSubsampled == None and\
					len(self.numFMsPerLayerSubsampled) <> len(self.numFMsPerLayerNormal) : #user specified subsampled layers.
				self.errorRequireKernelDimensionsSubsampled(self.kernDimPerLayerNormal, numFMsSubsampled)
			# kernDimSubsampled was specified. Now it's going to be tricky to make sure everything alright.
			elif not checkKernDimPerLayerCorrect3dAndNumLayers(kernDimSubsampled, len(self.numFMsPerLayerSubsampled)) :
				self.errReqKernDimNormalCorr()
			else : #kernel dimensions specified and are correct (3d, same number of layers as subsampled specified). Need to check the two receptive fields and make sure they are correct.
				self.kernDimPerLayerSubsampled = kernDimSubsampled
				self.receptiveFieldSubsampled = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(self.kernDimPerLayerSubsampled)
				if self.receptiveFieldNormal <> self.receptiveFieldSubsampled :
					self.errorReceptiveFieldsOfNormalAndSubsampledDifferent(self.receptiveFieldNormal, self.receptiveFieldSubsampled)
				#Everything alright, finally. Proceed safely...
			self.subsampleFactor = subsampleFactor if subsampleFactor <> None else [3,3,3]
			if len(self.subsampleFactor) <> 3 :
				self.errorSubFactor3d()
			if not checkSubsampleFactorEven(self.subsampleFactor) :
				self.warnSubFactorOdd()
			residConnAtLayersSubsampled = residConnAtLayersSubsampled if residConnAtLayersSubsampled <> None else residConnAtLayersNormal
		
		#==FC Layers==
		self.numFMsInExtraFcs = numFMsFc if numFMsFc <> None else []
		self.kernelDimensionsFirstFcLayer = kernelDimensionsFirstFcLayer if kernelDimensionsFirstFcLayer <> None else [1,1,1]
		assert len(self.kernelDimensionsFirstFcLayer) == 3 and (False not in [ dim > 0 for dim in self.kernelDimensionsFirstFcLayer] )
		residConnAtLayersFc = residConnAtLayersFc if residConnAtLayersFc <> None else []
						
		#==Size of Image Segments ==
		self.segmDimNormalTrain = segmDimTrain if segmDimTrain <> None else self.errReqSegmDimTrain()
		self.segmDimNormalVal = segmDimVal if segmDimVal <> None else self.receptiveFieldNormal
		self.segmDimNormalInfer = segmDimInfer if segmDimInfer <> None else self.segmDimNormalTrain
		for (tr0_val1_inf2, segmentDimensions) in [ (0,self.segmDimNormalTrain), (1,self.segmDimNormalVal), (2,self.segmDimNormalInfer) ] :
			if not checkReceptiveFieldFineInComparisonToSegmentSize(self.receptiveFieldNormal, segmentDimensions) :
				self.errorSegmDimensionsSmallerThanReceptiveF(self.receptiveFieldNormal, segmentDimensions, tr0_val1_inf2)


		#=== Batch Sizes ===
		self.batchSizeTrain = batchSizeTrain if batchSizeTrain <> None else self.errReqBatchSizeTr()
		self.batchSizeVal = batchSizeVal if batchSizeVal <> None else self.batchSizeTrain
		self.batchSizeInfer = batchSizeInfer if batchSizeInfer <> None else self.batchSizeTrain

		#=== Dropout rates ===
		self.dropNormal = dropNormal if dropNormal <> None else []
		self.dropSubsampled = dropSubsampled if dropSubsampled <> None else []
		self.dropFc = dropFc if dropFc <> None else self.defaultDropFcList(self.numFMsInExtraFcs) #default = [0.0, 0.5, ..., 0.5]
		self.dropoutRatesForAllPathways = [self.dropNormal, self.dropSubsampled, self.dropFc, []]

		#==Regularization==
		self.l1Reg = l1Reg if l1Reg <> None else 0.000001
		self.l2Reg = l2Reg if l2Reg <> None else 0.0001

		#== Weight Initialization==
		self.initialMethodClassic0Delving1 = initialMethod if initialMethod <> None else 1
		if not self.initialMethodClassic0Delving1 in [0,1]:
			self.errorReqInitializationMethod01()
		#== Activation Function ==
		self.activationFunctionRelu0Prelu1 = activationFunction if activationFunction <> None else 1
		if not self.activationFunctionRelu0Prelu1 in [0,1]:
			self.errorReqActivFunction01()

		#==BATCH NORMALIZATION==
		self.applyBnToInputOfPathways = [False, False, "Placeholder", False] # the 3 entry, for FC, is always True internally.
		self.bnRollingAverOverThatManyBatches = bnRollingAverOverThatManyBatches if bnRollingAverOverThatManyBatches <> None else 60
		
		#====Optimization=====
		self.learningRate = learningRate if learningRate <> None else 0.001
		self.optimizerSgd0Adam1Rms2 = optimizerSgd0Adam1Rms2 if optimizerSgd0Adam1Rms2 <> None else 2
		if self.optimizerSgd0Adam1Rms2 == 0 :
			self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder";
			self.rhoRms = "placeholder"; self.eRms = "placeholder";
		elif self.optimizerSgd0Adam1Rms2 == 1 :
			self.b1Adam = b1Adam if b1Adam <> None else 0.9 #default in paper and seems good
			self.b2Adam = b2Adam if b2Adam <> None else 0.999 #default in paper and seems good
			self.eAdam = eAdam if eAdam else 10**(-8)
			self.rhoRms = "placeholder"; self.eRms = "placeholder";
		elif self.optimizerSgd0Adam1Rms2 == 2 :
			self.b1Adam = "placeholder"; self.b2Adam = "placeholder"; self.eAdam = "placeholder";
			self.rhoRms = rhoRms if rhoRms <> None else 0.9 #default in paper and seems good
			self.eRms = eRms if eRms <> None else 10**(-4) # 1e-6 was the default in the paper, but blew up the gradients in first try. Never tried 1e-5 yet.
		else :
			self.errorRequireOptimizer012()

		self.classicMom0Nesterov1 = classicMom0Nesterov1 if classicMom0Nesterov1 <> None else 1
		if self.classicMom0Nesterov1 not in [0,1]:
			self.errorRequireMomentumClass0Nestov1()
		self.momNonNormalized0Normalized1 = momNonNormalized0Normalized1 if momNonNormalized0Normalized1 <> None else 1
		if self.momNonNormalized0Normalized1 not in [0,1] :
			self.errorRequireMomNonNorm0Norm1()
		self.momentumValue = momentumValue if momentumValue <> None else 0.6
		if self.momentumValue < 0. or self.momentumValue > 1:
			self.errorRequireMomValueBetween01()

		#==============CALCULATED=====================
		if self.useSubsampledBool :
			self.segmDimSubsampledTrain = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalTrain, self.receptiveFieldNormal, self.subsampleFactor)
			self.segmDimSubsampledVal = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalVal, self.receptiveFieldNormal, self.subsampleFactor)
			self.segmDimSubsampledInfer = calculateSubsampledImagePartDimensionsFromImagePartSizePatchSizeAndSubsampleFactor(self.segmDimNormalInfer, self.receptiveFieldNormal, self.subsampleFactor)
		else :
			self.segmDimSubsampledTrain = []; self.segmDimSubsampledVal = []; self.segmDimSubsampledInfer = [];

		# Residual Connections backwards, per pathway type :
		self.checkLayersForResidualsGivenDoNotInclude1st(residConnAtLayersNormal, residConnAtLayersSubsampled, residConnAtLayersFc)
		# The following variable passed to the system takes indices, ie number starts from 0. User specifies from 1.
		self.indicesOfLayersToConnectResidualsInOutput = [ 	[ layerNum - 1 for layerNum in residConnAtLayersNormal ],
														 	[ layerNum - 1 for layerNum in residConnAtLayersSubsampled ],
														 	[ layerNum - 1 for layerNum in residConnAtLayersFc ],
														 	[] 
														 ]

		#============= HIDDENS ======================
		self.costFunctionLetter = "L"

		self.numberOfInputChannelsSubsampled = self.numberOfInputChannelsNormal

		#----for the zoomed-in pathway----
		self.zoomedInPatchDimensions = [9, 9, 9]
		self.nkernsZoomedIn1 = [] #[15,15,15,30]
		self.kernelDimensionsZoomedIn1 = [[3,3,3], [3,3,3], [3,3,3], [3,3,3]]

		#MultiscaleConnections:
		self.convLayersToConnectToFirstFcForMultiscaleFromAllLayerTypes = [ [], [] ] #a sublist for each pathway. Starts from 0 index. Give a sublist, even empty for no connections.
		#... It's ok if I dont have a 2nd path but still give a 2nd sublist, it's controlled by nkernsSubsampled.

		#-------POOLING---------- (not fully supported currently)
		#One entry per pathway-type. leave [] if the pathway does not exist or there is no mp there AT ALL.
		#Inside each entry, put a list FOR EACH LAYER. It should be [] for the layer if no mp there. But FOR EACH LAYER.
		#MP is applied >>AT THE INPUT of the layer<<. To use mp to a layer, put a list of [[dsr,dsc,dsz], [strr,strc,strz], [mirrorPad-r,-c,-z], mode] which give the dimensions of the mp window, the stride, how many times to mirror the last slot at each dimension for padding (give 0 for none), the mode (usually 'max' pool). Eg [[2,2,2],[1,1,1]] or [[2,2,2],[2,2,2]] usually.
		self.maxPoolingParamsStructure = [ #If a pathway is not used, put an empty list in the first dimension entry. 
						[ [] for layeri in xrange(len(self.numFMsPerLayerNormal)) ], #[[[2,2,2], [1,1,1], [1,1,1], 'max'], [],[],[],[],[],[], []], #first pathway
						[ [] for layeri in xrange(len(self.numFMsPerLayerSubsampled)) ], #second pathway
						[ [] for layeri in xrange(len(self.numFMsInExtraFcs) + 1) ], #FC. This should NEVER be used for segmentation. Possible for classification though.
						[[],[],[],[]] #zoomed in pathway.
						]

		self.softmaxTemperature = 1.0 #Higher temperatures make the probabilities LESS distinctable. Actions have more similar probabilities. 
    def __init__(
            self,
            cnnModelName,
            sessionLogger,
            mainOutputAbsFolder,
            folderForSessionCnnModels,
            #===MODEL PARAMETERS===
            numberClasses,
            numberOfInputChannelsNormal,
            #===Normal pathway===
            numFMsNormal,
            kernDimNormal,
            residConnAtLayersNormal,
            lowerRankLayersNormal,
            #==Subsampled pathway==
            useSubsampledBool,
            numFMsSubsampled,
            kernDimSubsampled,
            subsampleFactor,
            residConnAtLayersSubsampled,
            lowerRankLayersSubsampled,
            #==FC Layers====
            numFMsFc,
            kernelDimensionsFirstFcLayer,
            residConnAtLayersFc,

            #==Size of Image Segments ==
            segmDimTrain,
            segmDimVal,
            segmDimInfer,
            #== Batch Sizes ==
            batchSizeTrain,
            batchSizeVal,
            batchSizeInfer,

            #===Other Architectural Parameters ===
            activationFunction,
            #==Dropout Rates==
            dropNormal,
            dropSubsampled,
            dropFc,

            #== Weight Initialization==
            initialMethod,

            #== Batch Normalization ==
            bnRollingAverOverThatManyBatches):

        #Importants for running session.

        self.cnnModelName = cnnModelName if cnnModelName else getDefaultModelName(
        )
        self.sessionLogger = sessionLogger
        self.mainOutputAbsFolder = mainOutputAbsFolder
        self.pathAndFilenameToSaveModel = os.path.abspath(
            folderForSessionCnnModels + "/" + self.cnnModelName)

        #===========MODEL PARAMETERS==========
        self.numberClasses = numberClasses if numberClasses <> None else self.errReqNumClasses(
        )
        self.numberOfInputChannelsNormal = numberOfInputChannelsNormal if numberOfInputChannelsNormal <> None or\
                                                        numberOfInputChannelsNormal<1 else self.errReqNumChannels()

        #===Normal pathway===
        self.numFMsPerLayerNormal = numFMsNormal if numFMsNormal <> None and numFMsNormal > 0 else self.errReqFMsNormal(
        )
        numOfLayers = len(self.numFMsPerLayerNormal)
        self.kernDimPerLayerNormal = kernDimNormal if checkKernDimPerLayerCorrect3dAndNumLayers(
            kernDimNormal, numOfLayers) else self.errReqKernDimNormal()
        self.receptiveFieldNormal = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(
            self.kernDimPerLayerNormal)  # Just for COMPATIBILITY CHECKS!
        residConnAtLayersNormal = residConnAtLayersNormal if residConnAtLayersNormal <> None else [
        ]  #layer number, starting from 1 for 1st layer. NOT indices.
        lowerRankLayersNormal = lowerRankLayersNormal if lowerRankLayersNormal <> None else [
        ]  #layer number, starting from 1 for 1st layer. NOT indices.

        #==Subsampled pathway==
        self.useSubsampledBool = useSubsampledBool if useSubsampledBool <> None else False
        if not self.useSubsampledBool:
            self.numFMsPerLayerSubsampled = []
            self.kernDimPerLayerSubsampled = []
            self.receptiveFieldSubsampled = []
            self.subsampleFactor = []
            residConnAtLayersSubsampled = []
            lowerRankLayersSubsampled = []

        else:
            self.numFMsPerLayerSubsampled = numFMsSubsampled if numFMsSubsampled <> None else self.numFMsPerLayerNormal
            self.numFMsPerLayerSubsampled = self.changeDatastructureToListOfListsForSecondaryPathwaysIfNeeded(
                self.numFMsPerLayerSubsampled)
            # check that all subsampled pathways have the same number of layers. Limitation in the code currently, because I use kernDimSubsampled for all of them.
            if not self.checkThatSublistsHaveSameLength(
                    self.numFMsPerLayerSubsampled):
                self.errReqSameNumOfLayersPerSubPathway()

            numOfLayersInEachSubPath = len(self.numFMsPerLayerSubsampled[0])
            if kernDimSubsampled == None and numOfLayersInEachSubPath == len(
                    self.numFMsPerLayerNormal):
                self.kernDimPerLayerSubsampled = self.kernDimPerLayerNormal
                self.receptiveFieldSubsampled = self.receptiveFieldNormal
            elif kernDimSubsampled == None and numOfLayersInEachSubPath <> len(
                    self.numFMsPerLayerNormal
            ):  #user specified subsampled layers.
                self.errorRequireKernelDimensionsSubsampled(
                    self.kernDimPerLayerNormal, numFMsSubsampled)
            # kernDimSubsampled was specified. Now it's going to be tricky to make sure everything alright.
            elif not checkKernDimPerLayerCorrect3dAndNumLayers(
                    kernDimSubsampled, numOfLayersInEachSubPath):
                self.errReqKernDimNormalCorr()
            else:  #kernel dimensions specified and are correct (3d, same number of layers as subsampled specified). Need to check the two receptive fields and make sure they are correct.
                self.kernDimPerLayerSubsampled = kernDimSubsampled
                self.receptiveFieldSubsampled = calculateReceptiveFieldDimensionsFromKernelsDimListPerLayerForFullyConvCnnWithStrides1(
                    self.kernDimPerLayerSubsampled)
                if self.receptiveFieldNormal <> self.receptiveFieldSubsampled:
                    self.errorReceptiveFieldsOfNormalAndSubsampledDifferent(
                        self.receptiveFieldNormal,
                        self.receptiveFieldSubsampled)
                #Everything alright, finally. Proceed safely...
            self.subsampleFactor = subsampleFactor if subsampleFactor <> None else [
                3, 3, 3
            ]
            self.subsampleFactor = self.changeDatastructureToListOfListsForSecondaryPathwaysIfNeeded(
                self.subsampleFactor)
            for secondaryPathway_i in xrange(
                    len(self.subsampleFactor)
            ):  #It should now be a list of lists, one sublist per secondary pathway. This is what is currently defining how many pathways to use.
                if len(self.subsampleFactor[secondaryPathway_i]) <> 3:
                    self.errorSubFactor3d()
                if not checkSubsampleFactorEven(
                        self.subsampleFactor[secondaryPathway_i]):
                    self.warnSubFactorOdd()
            #---For multiple lower-scale pathways, via the numFMsPerLayerSubsampled and subsampleFactor config ----
            numOfSubsPaths = max(len(self.numFMsPerLayerSubsampled),
                                 len(self.subsampleFactor))
            # Default behaviour:
            # If less sublists in numFMsPerLayerSubsampled were given than numOfSubsPaths, add more sublists of subsampleFactors, for the extra subpaths.
            for _ in range(numOfSubsPaths -
                           len(self.numFMsPerLayerSubsampled)):
                numFmsForLayersOfLastSubPath = self.numFMsPerLayerSubsampled[
                    -1]
                self.numFMsPerLayerSubsampled.append([
                    max(1, int(numFmsInLayer_i / 2))
                    for numFmsInLayer_i in numFmsForLayersOfLastSubPath
                ])  # half of how many previous subPath had.
            # If less sublists in subsampleFactor were given than numOfSubsPaths, add more sublists of subsampleFactors, for the extra subpaths.
            for _ in range(numOfSubsPaths - len(self.subsampleFactor)):
                self.subsampleFactor.append(
                    [
                        subFactorInDim_i + 2
                        for subFactorInDim_i in self.subsampleFactor[-1]
                    ]
                )  # Adds one more sublist, eg [5,5,5], which is the last subFactor, increased by +2 in all rcz dimensions.

            # Residuals and lower ranks.
            residConnAtLayersSubsampled = residConnAtLayersSubsampled if residConnAtLayersSubsampled <> None else residConnAtLayersNormal
            lowerRankLayersSubsampled = lowerRankLayersSubsampled if lowerRankLayersSubsampled <> None else lowerRankLayersNormal

        #==FC Layers==
        self.numFMsInExtraFcs = numFMsFc if numFMsFc <> None else []
        self.kernelDimensionsFirstFcLayer = kernelDimensionsFirstFcLayer if kernelDimensionsFirstFcLayer <> None else [
            1, 1, 1
        ]
        assert len(self.kernelDimensionsFirstFcLayer) == 3 and (False not in [
            dim > 0 for dim in self.kernelDimensionsFirstFcLayer
        ])
        residConnAtLayersFc = residConnAtLayersFc if residConnAtLayersFc <> None else []

        #==Size of Image Segments ==
        self.segmDimNormalTrain = segmDimTrain if segmDimTrain <> None else self.errReqSegmDimTrain(
        )
        self.segmDimNormalVal = segmDimVal if segmDimVal <> None else self.receptiveFieldNormal
        self.segmDimNormalInfer = segmDimInfer if segmDimInfer <> None else self.segmDimNormalTrain
        for (tr0_val1_inf2,
             segmentDimensions) in [(0, self.segmDimNormalTrain),
                                    (1, self.segmDimNormalVal),
                                    (2, self.segmDimNormalInfer)]:
            if not checkReceptiveFieldFineInComparisonToSegmentSize(
                    self.receptiveFieldNormal, segmentDimensions):
                self.errorSegmDimensionsSmallerThanReceptiveF(
                    self.receptiveFieldNormal, segmentDimensions,
                    tr0_val1_inf2)

        #=== Batch Sizes ===
        self.batchSizeTrain = batchSizeTrain if batchSizeTrain <> None else self.errReqBatchSizeTr(
        )
        self.batchSizeVal = batchSizeVal if batchSizeVal <> None else self.batchSizeTrain
        self.batchSizeInfer = batchSizeInfer if batchSizeInfer <> None else self.batchSizeTrain

        #=== Dropout rates ===
        self.dropNormal = dropNormal if dropNormal <> None else []
        self.dropSubsampled = dropSubsampled if dropSubsampled <> None else []
        self.dropFc = dropFc if dropFc <> None else self.defaultDropFcList(
            self.numFMsInExtraFcs)  #default = [0.0, 0.5, ..., 0.5]
        self.dropoutRatesForAllPathways = [
            self.dropNormal, self.dropSubsampled, self.dropFc, []
        ]

        #== Weight Initialization==
        self.initialMethodClassic0Delving1 = initialMethod if initialMethod <> None else 1
        if not self.initialMethodClassic0Delving1 in [0, 1]:
            self.errorReqInitializationMethod01()
        #== Activation Function ==
        self.activationFunctionRelu0Prelu1 = activationFunction if activationFunction <> None else 1
        if not self.activationFunctionRelu0Prelu1 in [0, 1]:
            self.errorReqActivFunction01()

        #==BATCH NORMALIZATION==
        self.applyBnToInputOfPathways = [
            False, False, True, False
        ]  # the 3 entry, for FC, should always be True.
        self.bnRollingAverOverThatManyBatches = bnRollingAverOverThatManyBatches if bnRollingAverOverThatManyBatches <> None else 60

        #==============CALCULATED=====================
        # Residual Connections backwards, per pathway type :
        self.checkLayersForResidualsGivenDoNotInclude1st(
            residConnAtLayersNormal, residConnAtLayersSubsampled,
            residConnAtLayersFc)
        # The following variable passed to the system takes indices, ie number starts from 0. User specifies from 1.
        self.indicesOfLayersToConnectResidualsInOutput = [
            [layerNum - 1 for layerNum in residConnAtLayersNormal],
            [layerNum - 1 for layerNum in residConnAtLayersSubsampled],
            [layerNum - 1 for layerNum in residConnAtLayersFc], []
        ]

        self.indicesOfLowerRankLayersPerPathway = [
            [layerNum - 1 for layerNum in lowerRankLayersNormal],
            [layerNum - 1 for layerNum in lowerRankLayersSubsampled],
            [],  #FC doesn't make sense to be lower rank. It's 1x1x1 anyway.
            []
        ]
        self.ranksOfLowerRankLayersForEachPathway = [[
            2 for layer_i in self.indicesOfLowerRankLayersPerPathway[0]
        ], [2 for layer_i in self.indicesOfLowerRankLayersPerPathway[1]], [],
                                                     []]
        #============= HIDDENS ======================
        self.numberOfInputChannelsSubsampled = self.numberOfInputChannelsNormal

        #MultiscaleConnections:
        self.convLayersToConnectToFirstFcForMultiscaleFromAllLayerTypes = [
            [], []
        ]  #a sublist for each pathway. Starts from 0 index. Give a sublist, even empty for no connections.
        #... It's ok if I dont have a 2nd path but still give a 2nd sublist, it's controlled by nkernsSubsampled.

        #-------POOLING---------- (not fully supported currently)
        #One entry per pathway-type. leave [] if the pathway does not exist or there is no mp there AT ALL.
        #Inside each entry, put a list FOR EACH LAYER. It should be [] for the layer if no mp there. But FOR EACH LAYER.
        #MP is applied >>AT THE INPUT of the layer<<. To use mp to a layer, put a list of [[dsr,dsc,dsz], [strr,strc,strz], [mirrorPad-r,-c,-z], mode] which give the dimensions of the mp window, the stride, how many times to mirror the last slot at each dimension for padding (give 0 for none), the mode (usually 'max' pool). Eg [[2,2,2],[1,1,1]] or [[2,2,2],[2,2,2]] usually.
        #If a pathway is not used (eg subsampled), put an empty list in the first dimension entry.
        mpParamsNorm = [
            [] for layeri in xrange(len(self.numFMsPerLayerNormal))
        ]  #[[[2,2,2], [1,1,1], [1,1,1], 'max'], [],[],[],[],[],[], []], #first pathway
        mpParamsSubs = [
            [] for layeri in xrange(len(self.numFMsPerLayerSubsampled[0]))
        ] if self.useSubsampledBool else [
        ]  # CAREFUL about the [0]. Only here till this structure is made different per pathway and not pathwayType.
        mpParamsFc = [
            [] for layeri in xrange(len(self.numFMsInExtraFcs) + 1)
        ]  #FC. This should NEVER be used for segmentation. Possible for classification though.
        self.maxPoolingParamsStructure = [
            mpParamsNorm, mpParamsSubs, mpParamsFc
        ]

        self.softmaxTemperature = 1.0  #Higher temperatures make the probabilities LESS distinctable. Actions have more similar probabilities.