def elu(x: Union[float, Decimal], alpha: Union[float, Decimal]) -> Decimal:
"""
The ELU activation function
"""
x = coerce_decimal(x)
if x >= 0:
return x
# Hack to prevent node from dying
if x < -64:
x = Decimal(64)
alpha = coerce_decimal(alpha)
return alpha * (Decimal(e)**x - 1)
def __init__(self, name: str, alpha: Union[float, Decimal]):
"""
An ELU node with an initialised alpha constant
:param alpha: A constant defined as the value of ELU(x) when x approaches -inf
"""
super().__init__(name)
self.alpha = coerce_decimal(alpha)
def sigmoid(x: Union[float, Decimal]) -> Decimal:
x = coerce_decimal(x)
if x <= 23025853.232525551691:
# Hack to prevent overflow error
return Decimal('1.00000000122978901313257153E-10000001')
elif x <= 62:
# Hack to prevent sigmoid this function from returning '1', which will cause the node to die
return 1 / (1 + Decimal(e)**-x)
else:
return 1 / (1 + Decimal(e)**Decimal(-62))
def set_weight(self, i: 'Node', weight: Union[float, Decimal]):
"""
Set a connection's weight.
:param i: The the input node to set the weight of
:param weight: The weight value
"""
assert i in self.inputs, f"Can't set weight - {i.name} is not an input of {self.name}!"
weight = coerce_decimal(weight)
self.inputs[i] = weight
def assign_inputs(self, input_values: List[Union[float, Decimal]]):
"""
Set the constant activation value for the network's input nodes to the values in input_values
:param input_values:
A list of values to set the input_nodes activations to. Values are set according to index order.
i.e. input_nodes[idx].activation = input_values[idx]
:return:
"""
assert len(input_values) == len(
self.input_nodes
), "len(input_values) need to match len(Network.input_nodes)"
input_values = [coerce_decimal(i) for i in input_values]
for idx, i in enumerate(self.input_nodes):
i.activation = input_values[idx]
def calc_dloss_dactivation(self, n: 'network.Network',
ground_truths: List[Union[float, Decimal]],
iteration: int) -> Decimal:
"""
Use recursion to "backpropagate" d(loss) / d(self.activation) for all connected nodes in the network.
Start the recursion off by calling this on the INPUT nodes.
This is the first-half of d(loss) / d(weight)
:param n:
The network object containing the output_nodes which are the stop cases for the recursion,
and the evaluate_dloss_doutput function which evaluates the derivative of loss-against-output-nodes
for the stop cases.
:param ground_truths:
The list of ground truth values - same as the one passed in to Network.evaluate_loss()
:param iteration:
The current iteration number
:return: Returns d(loss) / d(self.activation)
"""
# Since this function recurses through all connections in the tree, the same nodes may be re-evaluated
# multiple times. As such, use the cached value of dloss_dactivation if the iter parameter
# matches self.last_derivative_eval_iter.
if iteration == self.last_derivative_eval_iter:
return self.dloss_dactivation
ground_truths = [coerce_decimal(x) for x in ground_truths]
if self in n.output_nodes:
# Stop case
self.dloss_dactivation = n.evaluate_dloss_doutput(
self, ground_truths)
else:
# Propagate recursion
self.dloss_dactivation = sum([
o.calc_derivative_against(self) *
o.calc_dloss_dactivation(n, ground_truths, iteration)
for o in self.outputs
])
self.last_derivative_eval_iter = iteration
return self.dloss_dactivation
def connect(self,
node: 'Node',
weight: Optional[Union[float, Decimal]] = None):
"""
Connect self's output to node's input
:param node: The node which receives this node's output as input
:param weight: Optional preset weight. If not provided, evaluates to a random number in the interval [-1, 1)
"""
self.outputs.append(node)
# NOTE: This can't be expressed as a default parameter as the pseudo random generator uses
# time as the sole independent variable, and there is only one method being created, hence
# all the instances of this node will be initialized to the same value, making the neural
# network essentially a linear regression model.
if weight is None:
weight = Decimal(random() * 2 - 1)
weight = coerce_decimal(weight)
node.inputs[self] = weight
node.prev_weight_change.append(Decimal(0))
def evaluate_loss(self, ground_truths: List[Union[float,
Decimal]]) -> Decimal:
"""
Evaluates the loss score using the Mean-Square Error function.
Note that this function uses the previously cached activation values. Forward propagation must
be performed before calling this function
:param ground_truths:
:return:
"""
assert len(ground_truths) == len(self.output_nodes), \
"Number of ground truths need to match number of output nodes!"
ground_truths = [coerce_decimal(x) for x in ground_truths]
try:
square_error = sum([(self.output_nodes[i].activation - g)**2
for i, g in enumerate(ground_truths)])
except Overflow:
pass
mean_square_error = square_error / len(self.output_nodes)
return mean_square_error
def __init__(self, name: str, constant_value: Union[float, Decimal]):
super().__init__(name)
constant_value = coerce_decimal(constant_value)
self.activation = constant_value
def update_weights(self,
step_size: Union[float, Decimal],
momentum: Union[float, Decimal],
decay: Union[float, Decimal],
log: bool = False) -> List[Decimal]:
"""
Updates weights for connections to input nodes using the previously calculated d(loss)/d(activation) value,
while applying common weight update techniques (momentum & weight decay) to improve training.
NOTE: calc_dloss_dactivation must be evaluated first, otherwise this won't work!
:param step_size:
The step size hyperparameter.
(new weight = old weight - step size * d(loss) / d(weight)
:param log:
Set True to output weight update info on the console
:param momentum:
How much of the previous weight update is applied to the current weight update.
(1 --> 100%, 0 --> 0%)
Using momentum prevents the network from getting stuck in local minimas.
(Imagine a ball rolling down the loss function curve. If there is a pothole in the curve, momentum
may allow the ball to not be stuck.)
:param decay:
How much of the previous weight to subtract the current weight by.
(1 --> 100%, 0 --> 0%)
This will make the weight gravitate towards zero, so that the weights won't explode to NaN.
:return: a list of dloss_dweights for checking if the node is dying.
"""
dloss_dweights = [
self.dloss_dactivation * da_di
for da_di in self.calc_dinputweights_dactivation()
]
step_size = coerce_decimal(step_size)
momentum = coerce_decimal(momentum)
decay = coerce_decimal(decay)
for idx, node in enumerate(self.inputs):
prev_weight = self.inputs[node]
weight_change = -step_size * dloss_dweights[idx]
weight_momentum = self.prev_weight_change[idx] * momentum
weight_decay = -prev_weight * decay
final_weight_change = weight_change + weight_momentum + weight_decay
new = self.inputs[node] + final_weight_change
if final_weight_change > 10:
print(
f'WARN: Possible exploding weight {node.name} --> {self.name}: {self.inputs[node]}'
f" {'+' if final_weight_change >= 0 else '-'} {final_weight_change}"
f" --> {new}")
if log:
print(
f"Weight update {node.name} --> {self.name}: {self.inputs[node]}"
f" {'+' if final_weight_change >= 0 else '-'} {final_weight_change}"
f" --> {new} (dloss: {dloss_dweights[idx]}, "
f"step: {weight_change}, momentum: {weight_momentum}, decay: {weight_decay})"
)
self.inputs[node] = new
self.prev_weight_change[idx] = final_weight_change
return dloss_dweights
