def makeLayer( self,
rng,
layerConnected, # the basic layer, at the output of which to connect this softmax.
t = 1):
# t: temperature. Scalar
self._numberOfOutputClasses = layerConnected.getNumberOfFeatureMaps()
self._temperature = t
self._setBlocksInputAttributes(layerConnected.output["train"], layerConnected.output["val"], layerConnected.output["test"],
layerConnected.outputShape["train"], layerConnected.outputShape["val"], layerConnected.outputShape["test"])
# At this last classification layer, the conv output needs to have bias added before the softmax.
# NOTE: So, two biases are associated with this layer. self.b which is added in the ouput of the previous layer's output of conv,
# and this self._bClassLayer that is added only to this final output before the softmax.
(self._b,
logits_train,
logits_val,
logits_test) = makeBiasParamsAndApplyToFms( self.input["train"], self.input["val"], self.input["test"], self._numberOfOutputClasses )
self.params = self.params + [self._b]
# ============ Softmax ==============
self.p_y_given_x_train = tf.nn.softmax(logits_train/t, axis=1)
self.y_pred_train = tf.argmax(self.p_y_given_x_train, axis=1)
self.p_y_given_x_val = tf.nn.softmax(logits_val/t, axis=1)
self.y_pred_val = tf.argmax(self.p_y_given_x_val, axis=1)
self.p_y_given_x_test = tf.nn.softmax(logits_test/t, axis=1)
self.y_pred_test = tf.argmax(self.p_y_given_x_test, axis=1)
self._setBlocksOutputAttributes(self.p_y_given_x_train, self.p_y_given_x_val, self.p_y_given_x_test, self.inputShape["train"], self.inputShape["val"], self.inputShape["test"])
layerConnected.setTargetBlock(self)
def makeLayer(
self,
rng,
layerConnected, # the basic layer, at the output of which to connect this softmax.
softmaxTemperature=1):
self._numberOfOutputClasses = layerConnected.getNumberOfFeatureMaps()
self._softmaxTemperature = softmaxTemperature
self._setBlocksInputAttributes(layerConnected.output["train"],
layerConnected.output["val"],
layerConnected.output["test"],
layerConnected.outputShape["train"],
layerConnected.outputShape["val"],
layerConnected.outputShape["test"])
# At this last classification layer, the conv output needs to have bias added before the softmax.
# NOTE: So, two biases are associated with this layer. self.b which is added in the ouput of the previous layer's output of conv,
# and this self._bClassLayer that is added only to this final output before the softmax.
(self._b, biasedInputToSoftmaxTrain, biasedInputToSoftmaxVal,
biasedInputToSoftmaxTest) = makeBiasParamsAndApplyToFms(
self.input["train"], self.input["val"], self.input["test"],
self._numberOfOutputClasses)
self.params = self.params + [self._b]
# ============ Softmax ==============
#self.p_y_given_x_2d_train = ? Can I implement negativeLogLikelihood without this ?
(self.p_y_given_x_train,
self.y_pred_train) = applySoftmaxToFmAndReturnProbYandPredY(
biasedInputToSoftmaxTrain, self.inputShape["train"],
self._numberOfOutputClasses, softmaxTemperature)
(self.p_y_given_x_val,
self.y_pred_val) = applySoftmaxToFmAndReturnProbYandPredY(
biasedInputToSoftmaxVal, self.inputShape["val"],
self._numberOfOutputClasses, softmaxTemperature)
(self.p_y_given_x_test,
self.y_pred_test) = applySoftmaxToFmAndReturnProbYandPredY(
biasedInputToSoftmaxTest, self.inputShape["test"],
self._numberOfOutputClasses, softmaxTemperature)
self.network_output = biasedInputToSoftmaxTrain
self._setBlocksOutputAttributes(self.p_y_given_x_train,
self.p_y_given_x_val,
self.p_y_given_x_test,
self.inputShape["train"],
self.inputShape["val"],
self.inputShape["test"])
layerConnected.setTargetBlock(self)
def makeLayer(
self,
rng,
layerConnected, # the basic layer, at the output of which to connect this softmax.
t=1):
# t: temperature. Scalar
self._numberOfOutputClasses = layerConnected.getNumberOfFeatureMaps()
self._temperature = t
self._setBlocksInputAttributes(layerConnected.output["train"],
layerConnected.output["val"],
layerConnected.output["test"],
layerConnected.outputShape["train"],
layerConnected.outputShape["val"],
layerConnected.outputShape["test"])
# At this last classification layer, the conv output needs to have bias added before the softmax.
# NOTE: So, two biases are associated with this layer. self.b which is added in the ouput of the previous layer's output of conv,
# and this self._bClassLayer that is added only to this final output before the softmax.
(self._b, logits_train, logits_val,
logits_test) = makeBiasParamsAndApplyToFms(
self.input["train"], self.input["val"], self.input["test"],
self._numberOfOutputClasses)
self.params = self.params + [self._b]
# ============ Softmax ==============
self.p_y_given_x_train = tf.nn.softmax(logits_train / t, axis=1)
self.y_pred_train = tf.argmax(self.p_y_given_x_train, axis=1)
self.p_y_given_x_val = tf.nn.softmax(logits_val / t, axis=1)
self.y_pred_val = tf.argmax(self.p_y_given_x_val, axis=1)
self.p_y_given_x_test = tf.nn.softmax(logits_test / t, axis=1)
self.y_pred_test = tf.argmax(self.p_y_given_x_test, axis=1)
self._setBlocksOutputAttributes(self.p_y_given_x_train,
self.p_y_given_x_val,
self.p_y_given_x_test,
self.inputShape["train"],
self.inputShape["val"],
self.inputShape["test"])
layerConnected.setTargetBlock(self)
def _processInputWithBnNonLinearityDropoutPooling(
self,
rng,
inputToLayerTrain,
inputToLayerVal,
inputToLayerTest,
inputToLayerShapeTrain,
inputToLayerShapeVal,
inputToLayerShapeTest,
useBnFlag, # Must be true to do BN. Used to not allow doing BN on first layers straight on image, even if rollingAvForBnOverThayManyBatches > 0.
rollingAverageForBatchNormalizationOverThatManyBatches, #If this is <= 0, we are not using BatchNormalization, even if above is True.
activationFunc,
dropoutRate):
# ---------------- Order of what is applied -----------------
# Input -> [ BatchNorm OR biases applied] -> NonLinearity -> DropOut -> Pooling --> Conv ] # ala He et al "Identity Mappings in Deep Residual Networks" 2016
# -----------------------------------------------------------
#---------------------------------------------------------
#------------------ Batch Normalization ------------------
#---------------------------------------------------------
if useBnFlag and rollingAverageForBatchNormalizationOverThatManyBatches > 0:
self._appliedBnInLayer = True
self._rollingAverageForBatchNormalizationOverThatManyBatches = rollingAverageForBatchNormalizationOverThatManyBatches
(
inputToNonLinearityTrain,
inputToNonLinearityVal,
inputToNonLinearityTest,
self._gBn,
self._b,
# For rolling average :
self._muBnsArrayForRollingAverage,
self._varBnsArrayForRollingAverage,
self._sharedNewMu_B,
self._sharedNewVar_B,
self._newMu_B,
self._newVar_B) = applyBn(
rollingAverageForBatchNormalizationOverThatManyBatches,
inputToLayerTrain, inputToLayerVal, inputToLayerTest,
inputToLayerShapeTrain)
self.params = self.params + [self._gBn, self._b]
else: #Not using batch normalization
self._appliedBnInLayer = False
#make the bias terms and apply them. Like the old days before BN's own learnt bias terms.
numberOfInputChannels = inputToLayerShapeTrain[1]
(self._b, inputToNonLinearityTrain, inputToNonLinearityVal,
inputToNonLinearityTest) = makeBiasParamsAndApplyToFms(
inputToLayerTrain, inputToLayerVal, inputToLayerTest,
numberOfInputChannels)
self.params = self.params + [self._b]
#--------------------------------------------------------
#------------ Apply Activation/ non-linearity -----------
#--------------------------------------------------------
self._activationFunctionType = activationFunc
if self._activationFunctionType == "linear": # -1 stands for "no nonlinearity". Used for input layers of the pathway.
(inputToDropoutTrain, inputToDropoutVal,
inputToDropoutTest) = (inputToNonLinearityTrain,
inputToNonLinearityVal,
inputToNonLinearityTest)
elif self._activationFunctionType == "relu":
(inputToDropoutTrain, inputToDropoutVal,
inputToDropoutTest) = applyRelu(inputToNonLinearityTrain,
inputToNonLinearityVal,
inputToNonLinearityTest)
elif self._activationFunctionType == "prelu":
numberOfInputChannels = inputToLayerShapeTrain[1]
(self._aPrelu, inputToDropoutTrain, inputToDropoutVal,
inputToDropoutTest) = applyPrelu(inputToNonLinearityTrain,
inputToNonLinearityVal,
inputToNonLinearityTest,
numberOfInputChannels)
self.params = self.params + [self._aPrelu]
elif self._activationFunctionType == "elu":
(inputToDropoutTrain, inputToDropoutVal,
inputToDropoutTest) = applyElu(inputToNonLinearityTrain,
inputToNonLinearityVal,
inputToNonLinearityTest)
elif self._activationFunctionType == "selu":
(inputToDropoutTrain, inputToDropoutVal,
inputToDropoutTest) = applySelu(inputToNonLinearityTrain,
inputToNonLinearityVal,
inputToNonLinearityTest)
#------------------------------------
#------------- Dropout --------------
#------------------------------------
(inputToPoolTrain, inputToPoolVal, inputToPoolTest) = applyDropout(
rng, dropoutRate, inputToLayerShapeTrain, inputToDropoutTrain,
inputToDropoutVal, inputToDropoutTest)
#-------------------------------------------------------
#----------- Pooling ----------------------------------
#-------------------------------------------------------
if self._poolingParameters == []: #no max pooling before this conv
inputToConvTrain = inputToPoolTrain
inputToConvVal = inputToPoolVal
inputToConvTest = inputToPoolTest
inputToConvShapeTrain = inputToLayerShapeTrain
inputToConvShapeVal = inputToLayerShapeVal
inputToConvShapeTest = inputToLayerShapeTest
else: #Max pooling is actually happening here...
(inputToConvTrain,
inputToConvShapeTrain) = pool3dMirrorPad(inputToPoolTrain,
inputToLayerShapeTrain,
self._poolingParameters)
(inputToConvVal,
inputToConvShapeVal) = pool3dMirrorPad(inputToPoolVal,
inputToLayerShapeVal,
self._poolingParameters)
(inputToConvTest,
inputToConvShapeTest) = pool3dMirrorPad(inputToPoolTest,
inputToLayerShapeTest,
self._poolingParameters)
return (inputToConvTrain, inputToConvVal, inputToConvTest,
inputToConvShapeTrain, inputToConvShapeVal,
inputToConvShapeTest)
def _processInputWithBnNonLinearityDropoutPooling(self,
rng,
inputToLayerTrain,
inputToLayerVal,
inputToLayerTest,
inputToLayerShapeTrain,
inputToLayerShapeVal,
inputToLayerShapeTest,
useBnFlag, # Must be true to do BN. Used to not allow doing BN on first layers straight on image, even if rollingAvForBnOverThayManyBatches > 0.
movingAvForBnOverXBatches, #If this is <= 0, we are not using BatchNormalization, even if above is True.
activationFunc,
dropoutRate) :
# ---------------- Order of what is applied -----------------
# Input -> [ BatchNorm OR biases applied] -> NonLinearity -> DropOut -> Pooling --> Conv ] # ala He et al "Identity Mappings in Deep Residual Networks" 2016
# -----------------------------------------------------------
#---------------------------------------------------------
#------------------ Batch Normalization ------------------
#---------------------------------------------------------
if useBnFlag and movingAvForBnOverXBatches > 0 :
self._appliedBnInLayer = True
self._movingAvForBnOverXBatches = movingAvForBnOverXBatches
(inputToNonLinearityTrain,
inputToNonLinearityVal,
inputToNonLinearityTest,
self._gBn,
self._b,
# For rolling average :
self._muBnsArrayForRollingAverage,
self._varBnsArrayForRollingAverage,
self._sharedNewMu_B,
self._sharedNewVar_B,
self._newMu_B,
self._newVar_B
) = applyBn( movingAvForBnOverXBatches, inputToLayerTrain, inputToLayerVal, inputToLayerTest, inputToLayerShapeTrain)
self.params = self.params + [self._gBn, self._b]
# Create ops for updating the matrices with the bn inference stats.
self._op_update_mtrx_bn_inf_mu = tf.assign( self._muBnsArrayForRollingAverage[self._tf_plchld_int32], self._sharedNewMu_B )
self._op_update_mtrx_bn_inf_var = tf.assign( self._varBnsArrayForRollingAverage[self._tf_plchld_int32], self._sharedNewVar_B )
else : #Not using batch normalization
self._appliedBnInLayer = False
#make the bias terms and apply them. Like the old days before BN's own learnt bias terms.
numberOfInputChannels = inputToLayerShapeTrain[1]
(self._b,
inputToNonLinearityTrain,
inputToNonLinearityVal,
inputToNonLinearityTest) = makeBiasParamsAndApplyToFms( inputToLayerTrain, inputToLayerVal, inputToLayerTest, numberOfInputChannels )
self.params = self.params + [self._b]
#--------------------------------------------------------
#------------ Apply Activation/ non-linearity -----------
#--------------------------------------------------------
self._activationFunctionType = activationFunc
if self._activationFunctionType == "linear" : # -1 stands for "no nonlinearity". Used for input layers of the pathway.
( inputToDropoutTrain, inputToDropoutVal, inputToDropoutTest ) = (inputToNonLinearityTrain, inputToNonLinearityVal, inputToNonLinearityTest)
elif self._activationFunctionType == "relu" :
( inputToDropoutTrain, inputToDropoutVal, inputToDropoutTest ) = applyRelu(inputToNonLinearityTrain, inputToNonLinearityVal, inputToNonLinearityTest)
elif self._activationFunctionType == "prelu" :
numberOfInputChannels = inputToLayerShapeTrain[1]
( self._aPrelu, inputToDropoutTrain, inputToDropoutVal, inputToDropoutTest ) = applyPrelu(inputToNonLinearityTrain, inputToNonLinearityVal, inputToNonLinearityTest, numberOfInputChannels)
self.params = self.params + [self._aPrelu]
elif self._activationFunctionType == "elu" :
( inputToDropoutTrain, inputToDropoutVal, inputToDropoutTest ) = applyElu(inputToNonLinearityTrain, inputToNonLinearityVal, inputToNonLinearityTest)
elif self._activationFunctionType == "selu" :
( inputToDropoutTrain, inputToDropoutVal, inputToDropoutTest ) = applySelu(inputToNonLinearityTrain, inputToNonLinearityVal, inputToNonLinearityTest)
#------------------------------------
#------------- Dropout --------------
#------------------------------------
(inputToPoolTrain, inputToPoolVal, inputToPoolTest) = applyDropout(rng, dropoutRate, inputToLayerShapeTrain, inputToDropoutTrain, inputToDropoutVal, inputToDropoutTest)
#-------------------------------------------------------
#----------- Pooling ----------------------------------
#-------------------------------------------------------
if self._poolingParameters == [] : #no max pooling before this conv
inputToConvTrain = inputToPoolTrain
inputToConvVal = inputToPoolVal
inputToConvTest = inputToPoolTest
inputToConvShapeTrain = inputToLayerShapeTrain
inputToConvShapeVal = inputToLayerShapeVal
inputToConvShapeTest = inputToLayerShapeTest
else : #Max pooling is actually happening here...
(inputToConvTrain, inputToConvShapeTrain) = pool3dMirrorPad(inputToPoolTrain, inputToLayerShapeTrain, self._poolingParameters)
(inputToConvVal, inputToConvShapeVal) = pool3dMirrorPad(inputToPoolVal, inputToLayerShapeVal, self._poolingParameters)
(inputToConvTest, inputToConvShapeTest) = pool3dMirrorPad(inputToPoolTest, inputToLayerShapeTest, self._poolingParameters)
return (inputToConvTrain, inputToConvVal, inputToConvTest,
inputToConvShapeTrain, inputToConvShapeVal, inputToConvShapeTest )