def PIECE___average_half_square_error():
"""PIECE___average_half_squared_error
AVERAGE_HALF_SQUARE_ERROR = 1/2 x sum of squared differences between items in PREDICTED_OUTPUTS multi-dimensional
array and TARGET_OUTPUTS multi-dimensional array, divided by the number of cases
"""
forwards = {
'average_half_square_error': [
lambda a0, a1: ((a1 - a0)**2).sum() / (2 * a0.shape[0]), {
'a0': 'target_outputs',
'a1': 'predicted_outputs'
}
]
}
backwards = {
('DOVERD', 'average_half_square_error', 'predicted_outputs'): [
lambda a0, a1: (a1 - a0) / a0.shape[0], {
'a0': 'target_outputs',
'a1': 'predicted_outputs'
}
]
}
return Piece(forwards, backwards)
def PIECE___root_mean_square_error():
"""PIECE___root_mean_square_error:
ROOT_MEAN_SQUARE_ERROR = sum of squared differences between items in PREDICTED_OUTPUTS multi-dimensional
array and TARGET_OUTPUTS multi-dimensional array, divided by the number of cases, then square-rooted
"""
forwards = {
'root_mean_square_error': [
lambda a0, a1: sqrt(((a1 - a0)**2).sum() / a0.shape[0]), {
'a0': 'target_outputs',
'a1': 'predicted_outputs'
}
]
}
backwards = {
('DOVERD', 'root_mean_square_error', 'predicted_outputs'): [
lambda a0, a1, rmse: ((a1 - a0) / a0.shape[0]) / rmse, {
'a0': 'target_outputs',
'a1': 'predicted_outputs',
'rmse': 'root_mean_square_error'
}
]
}
return Piece(forwards, backwards)
def PIECE___average_binary_class_cross_entropy():
"""PIECE___average_binary_class_cross_entropy:
AVERAGE_BINARY_CLASS_CROSS_ENTROPY = cross entropy from TARGET_OUTPUTS multi-dimensional array (of numbers
between 0 and 1, inclusive) of PREDICTED_OUTPUTS multi-dimensional array (of numbers between 0 and 1, inclusive),
divided by the number of cases
"""
forwards = {
'average_binary_class_cross_entropy': [
lambda from_arr, of_arr, tiny=exp(-36): -(
(from_arr * log(of_arr + tiny)) + ((1. - from_arr) * log(
1. - of_arr + tiny))).sum() / from_arr.shape[0], {
'from_arr': 'target_outputs',
'of_arr': 'predicted_outputs'
}
]
}
backwards = {
('DOVERD', 'average_binary_class_cross_entropy', 'predicted_outputs'):
[
lambda from_arr, of_arr: -((from_arr / of_arr) - (
(1. - from_arr) / (1. - of_arr))) / from_arr.shape[0], {
'from_arr': 'target_outputs',
'of_arr': 'predicted_outputs'
}
]
}
return Piece(forwards, backwards)
def PIECE___rbm_energy():
forwards = 1
backwards = 1
return Piece(forwards, backwards)
def PIECE___average_multi_class_cross_entropy():
"""PIECE___average_multi_class_cross_entropy:
AVERAGE_MULTI_CLASS_CROSS_ENTROPY = cross entropy from TARGET_OUTPUTS matrix (of numbers between 0 and 1,
inclusive, with cases in rows) of PREDICTED_OUTPUTS matrix (of numbers between 0 and 1, inclusive, with cases in
rows), divided by the number of cases
"""
forwards = {
'average_multi_class_cross_entropy': [
lambda from_mat, of_mat, tiny=exp(-36): -(from_mat * log(
of_mat + tiny)).sum() / from_mat.shape[0], {
'from_mat': 'target_outputs',
'of_mat': 'predicted_outputs'
}
]
}
backwards = {
('DOVERD', 'average_multi_class_cross_entropy', 'predicted_outputs'): [
lambda from_mat, of_mat: -(from_mat / of_mat) / from_mat.shape[0],
{
'from_mat': 'target_outputs',
'of_mat': 'predicted_outputs'
}
]
}
return Piece(forwards, backwards)
def PIECE___equal():
"""PIECE___equal:
OUTPUTS multi-dimensional array = INPUTS multi-dimensional array
"""
forwards = {'outputs':
[lambda inp: inp,
{'inp': 'inputs'}]}
backwards = {('DOVERD', 'inputs'):
[lambda ddoutp: ddoutp,
{'ddoutp': ('DOVERD', 'outputs')}]}
return Piece(forwards, backwards)
def PIECE___tanh():
"""PIECE___tanh:
OUTPUTS multi-dimensional array = hyperbolic tangent function of INPUTS multi-dimensional array
"""
forwards = {'outputs':
[lambda inp: tanh(inp),
{'inp': 'inputs'}]}
backwards = {('DOVERD', 'inputs'):
[lambda ddoutp, outp: ddoutp * (1. - outp ** 2),
{'ddoutp': ('DOVERD', 'outputs'),
'outp': 'outputs'}]}
return Piece(forwards, backwards)
def PIECE___l1_weight_regularization():
"""PIECE___l1_weight_regularization:
WEIGHT_PENALTY = sum of absolute values of items of WEIGHTS multi-dimensional array
"""
forwards = {'weight_penalty': [lambda w: abs(w).sum(), {'w': 'weights'}]}
backwards = {
('DOVERD', 'weight_penalty', 'weights'):
[lambda w: sign(w), {
'w': 'weights'
}]
}
return Piece(forwards, backwards)
def PIECE___logistic():
"""PIECE___logistic:
OUTPUTS multi-dimensional array = logistic function of INPUTS multi-dimensional array
"""
forwards = {'outputs':
[lambda inp: 1. / (1. + exp(-inp)),
{'inp': 'inputs'}]}
backwards = {('DOVERD', 'inputs'):
[lambda ddoutp, outp: ddoutp * outp * (1. - outp),
{'ddoutp': ('DOVERD', 'outputs'),
'outp': 'outputs'}]}
return Piece(forwards, backwards)
def PIECE___logistic_with_temperature():
"""PIECE___logistic_with_temperature:
OUTPUTS multi-dimensional array = logistic function of INPUTS multi-dimensional array divided by temperature T
"""
forwards = {'outputs':
[lambda inp, temp: 1. / (1. + exp(- inp / temp)),
{'inp': 'inputs',
'temp': 'temperature'}]}
backwards = {('DOVERD', 'inputs'):
[lambda ddoutp, outp, temp: ddoutp * outp * (1. - outp) / temp,
{'ddoutp': ('DOVERD', 'outputs'),
'outp': 'outputs',
'temp': 'temperature'}]}
return Piece(forwards, backwards)
def PIECE___l2_weight_regularization():
"""PIECE___l2_weight_regularization:
WEIGHT_PENALTY = 1/2 x sum of squares of items of WEIGHTS multi-dimensional array
"""
forwards = {
'weight_penalty': [lambda w: (w**2).sum() / 2, {
'w': 'weights'
}]
}
backwards = {
('DOVERD', 'weight_penalty', 'weights'):
[lambda w: w, {
'w': 'weights'
}]
}
return Piece(forwards, backwards)
def PIECE___softmax():
"""PIECE___softmax:
OUTPUTS matrix (cases in rows) = softmax function of INPUTS matrix (cases in rows)
"""
from MBALearnsToCode.Functions import softmax, softmax_d_outputs_over_d_inputs,\
softmax_doverd_inputs_from_doverd_outputs
forwards = {'outputs':
[lambda inp: softmax(inp),
{'inp': 'inputs'}]}
backwards = {('DOVERD', 'inputs'):
[lambda ddoutp, outp:
softmax_doverd_inputs_from_doverd_outputs(ddoutp,
softmax_d_outputs_over_d_inputs(outputs = outp)),
{'ddoutp': ('DOVERD', 'outputs'),
'outp': 'outputs'}]}
return Piece(forwards, backwards)
def PIECE___matrix_product_of_inputs_and_weights(add_bias_column_to_inputs = True):
"""PIECE___matrix_product_of_inputs_and_weights:
OUTPUTS matrix (cases in rows) = matrix product of INPUTS matrix (cases in rows) and WEIGHTS matrix
"""
def add_bias_elements(array_a, nums_biases_to_add = [0, 1]):
a = array_a.copy()
for d in range(len(nums_biases_to_add)):
num_biases_to_add = nums_biases_to_add[d]
if num_biases_to_add > 0:
s = list(a.shape)
s[d] = num_biases_to_add
a = concatenate((ones(s), a), axis = d)
return a
def delete_bias_elements(array_a, nums_biases_to_delete = [1]):
a = array_a.copy()
for d in range(len(nums_biases_to_delete)):
num_biases_to_delete = nums_biases_to_delete[d]
if num_biases_to_delete > 0:
a = delete(a, range(num_biases_to_delete), axis = d)
return a
forwards = {'outputs':
[lambda inp, w: add_bias_elements(inp, [0, add_bias_column_to_inputs]).dot(w),
{'inp': 'inputs',
'w': 'weights'}]}
backwards = {('DOVERD', 'inputs'):
[lambda ddoutp, w: ddoutp.dot(delete_bias_elements(w, [add_bias_column_to_inputs]).T),
{'ddoutp': ('DOVERD', 'outputs'),
'w': 'weights'}],
('DOVERD', 'weights'):
[lambda ddoutp, inp: (add_bias_elements(inp, [0, add_bias_column_to_inputs]).T).dot(ddoutp),
{'ddoutp': ('DOVERD', 'outputs'),
'inp': 'inputs'}]}
return Piece(forwards, backwards)
def PIECE___root_mean_square_error_from_average_half_square_error():
"""PIECE___root_mean_square_error_from_average_half_square_error:
ROOT_MEAN_SQUARE_ERROR = square root of 2 x AVERAGE_HALF_SQUARE_ERROR
"""
forwards = {
'root_mean_square_error': [
lambda avg_half_se: sqrt(2 * avg_half_se), {
'avg_half_se': 'average_half_square_error'
}
]
}
backwards = {
('DOVERD', 'root_mean_square_error', 'average_half_square_error'):
[lambda rmse: 1. / rmse, {
'rmse': 'root_mean_square_error'
}]
}
return Piece(forwards, backwards)
def PIECE___from_vector_to_arrays(shapes___list, type = 'dict'):
"""PIECE___from_vector_to_arrays:
Converts a vector to dict/list/tuple of multi-dimensional arrays according to a list of shapes
"""
def from_arrays_to_vector(dict_or_list_or_tuple):
v = []
for i in range(len(dict_or_list_or_tuple)):
v += dict_or_list_or_tuple[i].flat
return array(v)
def from_vector_to_arrays(vector, shapes___list, type = 'dict'):
if type == 'dict':
arrays = {}
else:
arrays = len(shapes___list) * [[]]
v = vector.copy()
for i, s in enumerate(shapes___list):
if not isinstance(s, ndarray):
s = array(s)
num_elements = s.prod()
arrays[i] = v[:num_elements].reshape(s)
v = v[num_elements:]
if type == 'tuple':
arrays = tuple(arrays)
return arrays
forwards = {'arrays':
[lambda v: from_vector_to_arrays(v, shapes___list, type),
{'v': 'vector'}]}
backwards = {('DOVERD', 'vector'):
[lambda dda: from_arrays_to_vector(dda),
{'dda': ('DOVERD', 'arrays')}]}
return Piece(forwards, backwards)
def PIECE___average_unskewed_binary_class_cross_entropy():
"""PIECE___average_unskewed_binary_class_cross_entropy:
AVERAGE_UNSKEWED_BINARY_CLASS_CROSS_ENTROPY = cross entropy from TARGET_OUTPUTS multi-dimensional array (of numbers
between 0 and 1, inclusive) of PREDICTED_OUTPUTS multi-dimensional array (of numbers between 0 and 1, inclusive),
divided by the number of cases, adjusted by skewnesses between the positive (1) and negative (0) classes that are
represented by POSITIVE_CLASS_SKEWNESSES multi-dimensional array
"""
forwards = {
'average_unskewed_binary_class_cross_entropy': [
lambda from_arr, of_arr, pos_skew, tiny=exp(-36): -(
(from_arr * log(of_arr + tiny)) / pos_skew + (
(1. - from_arr) * log(1. - of_arr + tiny)) /
(2. - pos_skew)).sum() / from_arr.shape[0], {
'from_arr': 'target_outputs',
'of_arr': 'predicted_outputs',
'pos_skew': 'positive_class_skewnesses'
}
]
}
backwards = {
('DOVERD', 'average_unskewed_binary_class_cross_entropy', 'predicted_outputs'):
[
lambda from_arr, of_arr, pos_skew: -(
(from_arr / of_arr) / pos_skew - ((1. - from_arr) /
(1. - of_arr)) /
(2. - pos_skew)) / from_arr.shape[0], {
'from_arr': 'target_outputs',
'of_arr': 'predicted_outputs',
'pos_skew': 'positive_class_skewnesses'
}
]
}
return Piece(forwards, backwards)
def PIECE___average_unskewed_multi_class_cross_entropy():
"""PIECE___average_unskewed_multi_class_cross_entropy:
AVERAGE_UNSKEWED_MULTI_CLASS_CROSS_ENTROPY = cross entropy from TARGET_OUTPUTS matrix (of numbers between 0 and 1,
inclusive, with cases in rows) of PREDICTED_OUTPUTS matrix (of numbers between 0 and 1, inclusive, with cases in
rows), divided by the number of cases, adjusted by skewnesses among the classes that are represented by
CLASS_SKEWNESSES matrix
"""
forwards = {
'average_unskewed_multi_class_cross_entropy': [
lambda from_mat, of_mat, skew, tiny=exp(-36): -(
(from_mat * log(of_mat + tiny)) /
((from_mat.shape[1] / skew.sum(1, keepdims=True)) * skew)).sum(
) / from_mat.shape[0], {
'from_mat': 'target_outputs',
'of_mat': 'predicted_outputs',
'skew': 'multi_class_skewnesses'
}
]
}
backwards = {
('DOVERD', 'average_unskewed_multi_class_cross_entropy', 'predicted_outputs'):
[
lambda from_mat, of_mat, skew: -((from_mat / of_mat) / (
(from_mat.shape[1] / skew.sum(1, keepdims=True)) * skew)) /
from_mat.shape[0], {
'from_mat': 'target_outputs',
'of_mat': 'predicted_outputs',
'skew': 'multi_class_skewnesses'
}
]
}
return Piece(forwards, backwards)
