class SigmoidLayer(AbstractLayer):
"""
A layer which implements sigmoid activation
"""
def __init__(self, layerSize):
"""
A sigmoid layer can be connected to several inputs
"""
# Properly inherit the AbstractLayer
AbstractLayer.__init__(self)
# A sigmoid layer has an input port and output port
self.input = InputPort(layerSize)
self.output = OutputPort(layerSize)
def forward(self):
"""
Perform a forward step - activate the net input using logistic function
"""
# Perform the activation (logistic function)
self.output.setOutput(1.0 / (1.0 + gpu.exp(-self.input.getNetInput())))
def backward(self):
"""
Perform a backprop step - gradient is the derivative of the sigmoid functon
"""
self.input.setDelta(self.output.getOutput() * (1.0 - self.output.getOutput()) * self.output.getNetDelta())
class SigmoidLayer(AbstractLayer):
"""
A layer which implements sigmoid activation
"""
def __init__(self, layerSize):
"""
A sigmoid layer can be connected to several inputs
"""
# Properly inherit the AbstractLayer
AbstractLayer.__init__(self)
# A sigmoid layer has an input port and output port
self.input = InputPort(layerSize)
self.output = OutputPort(layerSize)
def forward(self):
"""
Perform a forward step - activate the net input using logistic function
"""
# Perform the activation (logistic function)
self.output.setOutput(1.0 / (1.0 + gpu.exp(-self.input.getNetInput())))
def backward(self):
"""
Perform a backprop step - gradient is the derivative of the sigmoid functon
"""
self.input.setDelta(self.output.getOutput() *
(1.0 - self.output.getOutput()) *
self.output.getNetDelta())
class AbsoluteValueLayer(AbstractLayer):
"""
A layer which implements absolute value activation
"""
def __init__(self, layerSize):
"""
An absolute value layer can be connected to several inputs
"""
# Properly inherit the AbstractLayer
AbstractLayer.__init__(self)
# A sigmoid layer has an input port and output port
self.input = InputPort(layerSize)
self.output = OutputPort(layerSize)
def forward(self):
"""
Perform a forward step - activate the net input using absolute value
"""
# Perform the activation (logistic function)
self.output.setOutput(np.abs(self.input.getNetInput()))
def backward(self):
"""
Perform a backprop step - gradient is the sign of the output
"""
self.input.setDelta(
np.sign(self.output.getOutput()) * self.output.getNetDelta())
class AbsoluteValueLayer(AbstractLayer):
"""
A layer which implements absolute value activation
"""
def __init__(self, layerSize):
"""
An absolute value layer can be connected to several inputs
"""
# Properly inherit the AbstractLayer
AbstractLayer.__init__(self)
# A sigmoid layer has an input port and output port
self.input = InputPort(layerSize)
self.output = OutputPort(layerSize)
def forward(self):
"""
Perform a forward step - activate the net input using absolute value
"""
# Perform the activation (logistic function)
self.output.setOutput(np.abs(self.input.getNetInput()))
def backward(self):
"""
Perform a backprop step - gradient is the sign of the output
"""
self.input.setDelta(np.sign(self.output.getOutput()) * self.output.getNetDelta())
class ReluLayer(AbstractLayer):
"""
A layer which implements rectified linear activation
"""
def __init__(self, layerSize):
"""
"""
# Properly inherit the AbstractLayer
AbstractLayer.__init__(self)
# A ReLU layer has an input port and output port
self.input = InputPort(layerSize)
self.output = OutputPort(layerSize)
def forward(self):
"""
Perform a forward step - activate the net input using the soft ReLU function
"""
# Perform the activation (set any negative values to zero)
self.output.setOutput(np.fmax(0.0, self.input.getNetInput()))
def backward(self):
"""
Perform a backprop step - gradient is simply 1 where the data is positive
"""
self.input.setDelta(
np.where(self.output.getOutput() > 0, 1.0, 0.0) *
self.output.getNetDelta())
class ReluLayer(AbstractLayer):
"""
A layer which implements rectified linear activation
"""
def __init__(self, layerSize):
"""
"""
# Properly inherit the AbstractLayer
AbstractLayer.__init__(self)
# A ReLU layer has an input port and output port
self.input = InputPort(layerSize)
self.output = OutputPort(layerSize)
def forward(self):
"""
Perform a forward step - activate the net input using the soft ReLU function
"""
# Perform the activation (set any negative values to zero)
self.output.setOutput(np.fmax(0.0, self.input.getNetInput()))
def backward(self):
"""
Perform a backprop step - gradient is simply 1 where the data is positive
"""
self.input.setDelta(np.where(self.output.getOutput() > 0, 1.0, 0.0) * self.output.getNetDelta())
class SoftmaxLayer(AbstractLayer):
"""
A layer which implements sigmoid activation
"""
def __init__(self, layerSize):
"""
A sigmoid layer can be connected to several inputs
"""
# Properly inherit the AbstractLayer
AbstractLayer.__init__(self)
# A sigmoid layer has an input port and output port
self.input = InputPort(layerSize)
self.output = OutputPort(layerSize)
def forward(self):
"""
Perform a forward step - activate the net input using logistic function
"""
# Perform the activation
net_input = self.input.getNetInput()
net_input[net_input > 10.0] = 10.0
net_input[net_input < -10.0] = -10.0
self.output.setOutput(np.exp(net_input) + 1e-10)
self.output.setOutput(self.output.getOutput() / (np.array([np.sum(self.output.getOutput(),1)]).transpose()))
def backward(self):
"""
Perform a backprop step - gradient is the derivative of the sigmoid functon
"""
# self.input.setDelta(self.output.getOutput() * (1.0 - self.output.getOutput()) * self.output.getNetDelta())
self.input.setDelta(self.output.getNetDelta())
