def gradient_numerical(
self,
h: Optional[TYPE_FLOAT] = None
) -> List[Union[TYPE_FLOAT, np.ndarray]]:
"""Calculate numerical gradients
Args:
h: small number for delta to calculate the numerical gradient
Returns:
[dX, dW]: Numerical gradients for X and W without bias
dX is dL/dX of shape (N, D-1) without the bias to match the original input
dW is dL/dW of shape (M, D) including the bias weight w0.
"""
name = "gradient_numerical"
self.logger.debug("layer[%s].%s", self.name, name)
L = self.objective
WT = self.W.T
def objective_X(x: np.ndarray):
return L(x @ WT)
def objective_W(w: np.ndarray):
return L(self.X @ w.T)
dX = numerical_jacobian(objective_X, self.X, delta=h)
dX = dX[::, 1:: # Omit the bias
]
dW = numerical_jacobian(objective_W, self.W, delta=h)
return [dX, dW]
def test_test_010_softmax_cross_entropy_log_loss_performance():
return
for _ in range(NUM_MAX_TEST_TIMES):
N = NUM_MAX_BATCH_SIZE
M = NUM_MAX_NODES
X = np.random.randn(N, M)
T = np.random.randint(0, M, N).astype(TYPE_LABEL)
f = partial(softmax_cross_entropy_log_loss, T=T)
def objective(x):
J, P = f(x)
return np.sum(J)
numerical_jacobian(objective, X)
softmax_cross_entropy_log_loss(X, T)
def gradient_numerical(self,
h: float = 1e-5) -> List[Union[float, np.ndarray]]:
"""Calculate numerical gradients
Args:
h: small number for delta to calculate the numerical gradient
Returns:
dX: [L(f(X+h) - L(f(X-h)] / 2h
"""
# L = Li(f(arg))
def L(X: np.ndarray):
# pylint: disable=not-callable
return self.objective(self.function(X))
dX = numerical_jacobian(L, self.X)
return [dX]
def test_020_numerical_jacobian_sigmoid(caplog):
"""Test Case for numerical gradient calculation
The domain of X is -BOUNDARY_SIGMOID < x < BOUNDARY_SIGMOID
Args:
u: Acceptable threshold value
"""
u: TYPE_FLOAT = GRADIENT_DIFF_ACCEPTANCE_VALUE
# y=sigmoid(x) -> dy/dx = y(1-y)
# 0.5 = sigmoid(0) -> dy/dx = 0.25
for _ in range(NUM_MAX_TEST_TIMES):
x = np.random.uniform(low=-BOUNDARY_SIGMOID,
high=BOUNDARY_SIGMOID,
size=1).astype(TYPE_FLOAT)
y = sigmoid(x)
analytical = np.multiply(y, (1 - y))
numerical = numerical_jacobian(sigmoid, x)
difference = np.abs(analytical - numerical)
acceptance = np.abs(analytical * GRADIENT_DIFF_ACCEPTANCE_RATIO)
assert np.all(difference < max(u, acceptance)), \
f"Needs difference < {max(u, acceptance)} but {difference}\nx is {x}"
def test_020_numerical_jacobian_avg(caplog):
"""Test Case for numerical gradient calculation for average function
A Jacobian matrix whose element 1/N is expected where N is X.size
"""
def f(X: np.ndarray):
return np.average(X)
# caplog.set_level(logging.DEBUG, logger=Logger.name)
u: TYPE_FLOAT = GRADIENT_DIFF_ACCEPTANCE_VALUE
for _ in range(NUM_MAX_TEST_TIMES):
# Batch input X of shape (N, M)
n = np.random.randint(low=1, high=NUM_MAX_BATCH_SIZE)
m = np.random.randint(low=1, high=NUM_MAX_NODES)
X = np.random.randn(n, m).astype(TYPE_FLOAT)
# Expected gradient matrix of shape (N,M) as label T
T = np.full(X.shape, 1 / X.size).astype(TYPE_FLOAT)
# Jacobian matrix of shame (N,M), each element of which is df/dXij
J = numerical_jacobian(f, X)
assert np.allclose(T, J, atol=u), \
f"(T - Z) < {u} is expected but {np.abs(T - J)}."
def disabled_test_040_softmax_log_loss_2d(caplog):
"""
TODO: Disabled as need to redesign numerical_jacobian for 32 bit floating.
Objective:
Verify the forward path constraints:
1. Layer output L/loss is np.sum(softmax_cross_entropy_log_loss) / N.
2. gradient_numerical() == numerical_jacobian(objective, X).
Verify the backward path constraints:
1. Analytical gradient G: gradient() == (P-1)/N
2. Analytical gradient G is close to GN: gradient_numerical().
"""
caplog.set_level(logging.DEBUG)
# --------------------------------------------------------------------------------
# Instantiate a CrossEntropyLogLoss layer
# --------------------------------------------------------------------------------
name = "test_040_softmax_log_loss_2d_ohe"
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES) # number of node > 1
_layer = layer.CrossEntropyLogLoss(
name=name,
num_nodes=M,
log_loss_function=softmax_cross_entropy_log_loss,
log_level=logging.DEBUG)
# ================================================================================
# Layer forward path
# ================================================================================
X = np.random.randn(N, M).astype(TYPE_FLOAT)
T = np.zeros_like(X, dtype=TYPE_LABEL) # OHE labels.
T[np.arange(N), np.random.randint(0, M, N)] = int(1)
# log_loss function require (X, T) in X(N, M), and T(N, M) in index label format.
X, T = transform_X_T(X, T)
_layer.T = T
Logger.debug("%s: X is \n%s\nT is \n%s", name, X, T)
# --------------------------------------------------------------------------------
# Expected analytical gradient EG = (dX/dL) = (A-T)/N
# --------------------------------------------------------------------------------
A = softmax(X)
EG = np.copy(A)
EG[np.arange(N), T] -= TYPE_FLOAT(
1) # Shape(N,), subtract from elements for T=1 only
EG /= TYPE_FLOAT(N)
# --------------------------------------------------------------------------------
# Total loss Z = np.sum(J)/N
# Expected loss EL = -sum(T*log(_A))
# (J, P) = softmax_cross_entropy_log_loss(X, T) and J:shape(N,) where J:shape(N,)
# is loss for each input and P is activation by sigmoid(X).
# --------------------------------------------------------------------------------
L = _layer.function(X)
J, P = softmax_cross_entropy_log_loss(X, T)
EL = np.array(-np.sum(logarithm(A[np.arange(N), T])) / N,
dtype=TYPE_FLOAT)
# Constraint: A == P as they are sigmoid(X)
assert np.all(np.abs(A-P) < ACTIVATION_DIFF_ACCEPTANCE_VALUE), \
f"Need A==P==sigmoid(X) but A=\n{A}\n P=\n{P}\n(A-P)=\n{(A-P)}\n"
# Constraint: Log loss layer output L == sum(J) from the log loss function
Z = np.array(np.sum(J) / N, dtype=TYPE_FLOAT)
assert np.array_equal(L, Z), \
f"Need log loss layer output L == sum(J) but L=\n{L}\nZ=\n{Z}."
# Constraint: L/loss is close to expected loss EL.
assert np.all(np.abs(EL-L) < LOSS_DIFF_ACCEPTANCE_VALUE), \
"Need EL close to L but \nEL=\n{EL}\nL=\n{L}\n"
# constraint: gradient_numerical() == numerical_jacobian(objective, X)
# TODO: compare the diff to accommodate numerical errors.
GN = _layer.gradient_numerical() # [dL/dX] from the layer
def objective(x):
"""Function to calculate the scalar loss L for cross entropy log loss"""
j, p = softmax_cross_entropy_log_loss(x, T)
return np.array(np.sum(j) / N, dtype=TYPE_FLOAT)
EGN = numerical_jacobian(objective, X) # Expected numerical dL/dX
assert np.array_equal(GN[0], EGN), \
f"GN[0]==EGN expected but GN[0] is \n%s\n EGN is \n%s\n" % (GN[0], EGN)
# ================================================================================
# Layer backward path
# ================================================================================
# constraint: Analytical gradient G: gradient() == EG == (P-1)/N.
dY = TYPE_FLOAT(1)
G = _layer.gradient(dY)
assert np.all(np.abs(G-EG) <= GRADIENT_DIFF_ACCEPTANCE_VALUE), \
f"Layer gradient dL/dX \n{G} \nneeds to be \n{EG}."
# constraint: Analytical gradient G is close to GN: gradient_numerical().
assert \
np.all(np.abs(G - GN[0]) <= GRADIENT_DIFF_ACCEPTANCE_VALUE) or \
np.all(np.abs(G - GN[0]) <= np.abs(GRADIENT_DIFF_ACCEPTANCE_RATIO * GN[0])), \
f"dX is \n{G}\nGN[0] is \n{GN[0]}\nRatio * GN[0] is \n{GRADIENT_DIFF_ACCEPTANCE_RATIO * GN[0]}.\n"
# constraint: Gradient g of the log loss layer needs -1 < g < 1
# abs(P-T) = abs(sigmoid(X)-T) cannot be > 1.
assert np.all(np.abs(G) < 1), \
f"Log loss layer gradient cannot be < -1 nor > 1 but\n{G}"
assert np.all(np.abs(GN[0]) < (1+GRADIENT_DIFF_ACCEPTANCE_RATIO)), \
f"Log loss layer gradient cannot be < -1 nor > 1 but\n{GN[0]}"
profiler.disable()
profiler.print_stats(sort="cumtime")
def disabled_test_030_objective_methods_2d_ohe():
"""
TODO: Disabled as need to redesign numerical_jacobian for 32 bit floating.
Objective:
Verify the forward path constraints:
1. Layer output L/loss is np.sum(cross_entropy_log_loss(softmax(X), T)) / N.
2. gradient_numerical() == numerical Jacobian numerical_jacobian(O, X).
Verify the backward path constraints:
1. Analytical gradient G: gradient() == (P-1)/N
2. Analytical gradient G is close to GN: gradient_numerical().
Expected:
Initialization detects the access to the non-initialized parameters and fails.
"""
def objective(X: np.ndarray) -> Union[float, np.ndarray]:
"""Dummy objective function to calculate the loss L"""
assert X.ndim == 0, "The output of the log loss should be of shape ()"
return X
# --------------------------------------------------------------------------------
# Instantiate a CrossEntropyLogLoss layer
# --------------------------------------------------------------------------------
name = "test_030_objective_methods_2d_ohe"
for _ in range(NUM_MAX_TEST_TIMES):
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
assert M >= 2, "Softmax is for multi label classification. "\
" Use Sigmoid for binary classification."
_layer = layer.CrossEntropyLogLoss(name=name,
num_nodes=M,
log_level=logging.DEBUG)
_layer.objective = objective
# ================================================================================
# Layer forward path
# ================================================================================
X = np.random.randn(N, M).astype(TYPE_FLOAT)
T = np.zeros_like(X, dtype=TYPE_LABEL) # OHE labels.
T[np.arange(N), np.random.randint(0, M, N)] = TYPE_LABEL(1)
_layer.T = T
Logger.debug("%s: X is \n%s\nT is \n%s", name, X, T)
P = softmax(X)
EG = (P - T) / N # Expected analytical gradient dL/dX = (P-T)/N
# --------------------------------------------------------------------------------
# constraint: L/loss == np.sum(cross_entropy_log_loss(softmax(X), T)) / N.
# --------------------------------------------------------------------------------
L = _layer.function(X)
Z = np.array(np.sum(cross_entropy_log_loss(softmax(X), T))) / N
assert np.array_equal(
L, Z), f"SoftmaxLogLoss output should be {L} but {Z}."
# --------------------------------------------------------------------------------
# constraint: gradient_numerical() == numerical Jacobian numerical_jacobian(O, X)
# --------------------------------------------------------------------------------
GN = _layer.gradient_numerical() # [dL/dX] from the _layer
# --------------------------------------------------------------------------------
# DO not use CrossEntropyLogLoss.function() to simulate the objective function for
# the expected GN. See the same part in test_030_objective_methods_1d_ohe().
# --------------------------------------------------------------------------------
# dummy= CrossEntropyLogLoss(
# name=name,
# num_nodes=M,
# log_level=logging.DEBUG
# )
# dummy.T = T
# dummy.objective = objective
# O = lambda x: dummy.objective(dummy.function(x)) # Objective function
O = lambda x: np.sum(cross_entropy_log_loss(softmax(x), T)) / N
# --------------------------------------------------------------------------------
EGN = numerical_jacobian(O, X) # Expected numerical dL/dX
assert np.array_equal(GN[0], EGN), \
f"GN[0]==EGN expected but GN[0] is \n%s\n EGN is \n%s\n" % (GN[0], EGN)
# ================================================================================
# Layer backward path
# ================================================================================
# --------------------------------------------------------------------------------
# constraint: Analytical gradient G: gradient() == (P-1)/N.
# --------------------------------------------------------------------------------
dY = float(1)
G = _layer.gradient(dY)
assert np.all(abs(G-EG) <= GRADIENT_DIFF_ACCEPTANCE_VALUE), \
f"Layer gradient dL/dX \n{G} \nneeds to be \n{EG}."
# --------------------------------------------------------------------------------
# constraint: Analytical gradient G is close to GN: gradient_numerical().
# --------------------------------------------------------------------------------
assert \
np.all(np.abs(G - GN[0]) <= GRADIENT_DIFF_ACCEPTANCE_VALUE) or \
np.all(np.abs(G - GN[0]) <= np.abs(GRADIENT_DIFF_ACCEPTANCE_RATIO * GN[0])), \
f"dX is \n{G}\nGN[0] is \n{GN[0]}\nRatio * GN[0] is \n{GRADIENT_DIFF_ACCEPTANCE_RATIO * GN[0]}.\n"
def test_030_objective_methods_1d_ohe():
"""
Objective:
Verify the forward path constraints:
1. Layer output L/loss is np.sum(cross_entropy_log_loss(softmax(X), T)) / N.
2. gradient_numerical() == numerical Jacobian numerical_jacobian(O, X).
Verify the backward path constraints:
1. Analytical gradient G: gradient() == (P-1)/N
2. Analytical gradient G is close to GN: gradient_numerical().
Expected:
Initialization detects the access to the non-initialized parameters and fails.
For X.ndim > 0, the layer transform X into 2D so as to use the numpy tuple-
like indexing:
P[
(0,3),
(2,4)
]
Hence, the shape of GN, G are 2D.
"""
# --------------------------------------------------------------------------------
# Instantiate a CrossEntropyLogLoss layer
# --------------------------------------------------------------------------------
name = "test_030_objective_methods_1d_ohe"
N = 1
for _ in range(NUM_MAX_TEST_TIMES):
M: int = np.random.randint(2, NUM_MAX_NODES)
assert M >= 2, "Softmax is for multi label classification. "\
" Use Sigmoid for binary classification."
_layer = layer.CrossEntropyLogLoss(name=name,
num_nodes=M,
log_level=logging.DEBUG)
# ================================================================================
# Layer forward path
# ================================================================================
X = np.random.randn(M).astype(TYPE_FLOAT)
T = np.zeros_like(X, dtype=TYPE_LABEL) # OHE labels.
T[np.random.randint(0, M)] = TYPE_LABEL(1)
_layer.T = T
P = softmax(X)
EG = ((P - T) / N).reshape(1, -1).astype(
TYPE_FLOAT) # Expected analytical gradient dL/dX = (P-T)/N
Logger.debug("%s: X is \n%s\nT is %s\nP is %s\nEG is %s\n", name, X, T,
P, EG)
# --------------------------------------------------------------------------------
# constraint: L/loss == np.sum(cross_entropy_log_loss(softmax(X), T)) / N.
# --------------------------------------------------------------------------------
L = _layer.function(X)
Z = np.array(np.sum(cross_entropy_log_loss(softmax(X), T)),
dtype=TYPE_FLOAT) / TYPE_FLOAT(N)
assert np.array_equal(
L, Z), f"SoftmaxLogLoss output should be {L} but {Z}."
# --------------------------------------------------------------------------------
# constraint: gradient_numerical() == numerical Jacobian numerical_jacobian(O, X)
# Use a dummy _layer for the objective function because using the "_layer"
# updates the X, Y which can interfere the independence of the _layer.
# --------------------------------------------------------------------------------
GN = _layer.gradient_numerical() # [dL/dX] from the _layer
# --------------------------------------------------------------------------------
# Cannot use CrossEntropyLogLoss.function() to simulate the objective function L.
# because it causes applying transform_X_T multiple times.
# Because internally transform_X_T(X, T) has transformed T into the index label
# in 1D with with length 1 by "T = T.reshape(-1)".
# Then providing X in 1D into "dummy.function(x)" re-run "transform_X_T(X, T)"
# again. The (X.ndim == T.ndim ==1) as an input and T must be OHE label for such
# combination and T.shape == P.shape must be true for OHE labels.
# However, T has been converted into the index format already by transform_X_T
# (applying transform_X_T multiple times) and (T.shape=(1,1), X.shape=(1, > 1)
# that violates the (X.shape == T.shape) constraint.
# --------------------------------------------------------------------------------
# dummy = CrossEntropyLogLoss(
# name="dummy",
# num_nodes=M,
# log_level=logging.DEBUG
# )
# dummy.T = T
# dummy.objective = objective
# dummy.function(X)
# --------------------------------------------------------------------------------
# O = lambda x: dummy.objective(dummy.function(x)) # Objective function
O = lambda x: np.sum(cross_entropy_log_loss(softmax(x), T),
dtype=TYPE_FLOAT) / TYPE_FLOAT(N)
# --------------------------------------------------------------------------------
EGN = numerical_jacobian(O, X).reshape(1,
-1) # Expected numerical dL/dX
assert np.array_equal(GN[0], EGN), \
f"Layer gradient_numerical GN \n{GN} \nneeds to be \n{EGN}."
# ================================================================================
# Layer backward path
# ================================================================================
# --------------------------------------------------------------------------------
# constraint: Analytical gradient G: gradient() == (P-1)/N.
# --------------------------------------------------------------------------------
dY = TYPE_FLOAT(1)
G = _layer.gradient(dY)
assert np.all(np.abs(G-EG) <= GRADIENT_DIFF_ACCEPTANCE_VALUE), \
f"Layer gradient dL/dX \n{G} \nneeds to be \n{EG} but G-EG \n{np.abs(G-EG)}\n"
# --------------------------------------------------------------------------------
# constraint: Analytical gradient G is close to GN: gradient_numerical().
# --------------------------------------------------------------------------------
assert \
np.all(np.abs(G - GN[0]) <= GRADIENT_DIFF_ACCEPTANCE_VALUE) or \
np.all(np.abs(G-GN[0]) <= np.abs(GRADIENT_DIFF_ACCEPTANCE_RATIO * GN[0])), \
f"dX is \n{G}\nGN[0] is \n{GN[0]}\nRatio * GN[0] is \n{GRADIENT_DIFF_ACCEPTANCE_RATIO * GN[0]}.\n"
def disabled_test_020_matmul_round_trip():
"""
TODO: Disabled as need to re-design numerical_jacobian for 32 bit float e.g TF.
Objective:
Verify the forward and backward paths at matmul.
Expected:
Forward path:
1. Matmul function(X) == X @ W.T
2. Numerical gradient should be the same with numerical Jacobian
Backward path:
3. Analytical gradient dL/dX == dY @ W
4. Analytical dL/dW == X.T @ dY
5. Analytical gradients are similar to the numerical gradient ones
Gradient descent
6. W is updated via the gradient descent.
7. Objective L is decreasing via the gradient descent.
"""
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
# --------------------------------------------------------------------------------
# Instantiate a Matmul layer
# --------------------------------------------------------------------------------
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(1, NUM_MAX_NODES)
D: int = np.random.randint(1, NUM_MAX_FEATURES)
W = weights.he(M, D + 1)
name = "test_020_matmul_methods"
def objective(X: np.ndarray) -> Union[float, np.ndarray]:
"""Dummy objective function to calculate the loss L"""
return np.sum(X)
# Test both static instantiation and build()
if TYPE_FLOAT(np.random.uniform()) < 0.5:
matmul = Matmul(name=name,
num_nodes=M,
W=W,
log_level=logging.DEBUG)
else:
matmul_spec = {
_NAME: "test_020_matmul_builder_to_fail_matmul_spec",
_NUM_NODES: M,
_NUM_FEATURES: D,
_WEIGHTS: {
_SCHEME: "he",
},
_OPTIMIZER: {
_SCHEME: "sGd"
}
}
matmul = Matmul.build(matmul_spec)
matmul.objective = objective
# ================================================================================
# Layer forward path
# Calculate the layer output Y=f(X), and get the loss L = objective(Y)
# Test the numerical gradient dL/dX=matmul.gradient_numerical().
#
# Note that bias columns are added inside the matmul layer instance, hence
# matmul.X.shape is (N, 1+D), matmul.W.shape is (M, 1+D)
# ================================================================================
X = np.random.randn(N, D).astype(TYPE_FLOAT)
Logger.debug("%s: X is \n%s", name, X)
# pylint: disable=not-callable
Y = matmul.function(X)
# pylint: disable=not-callable
L = matmul.objective(Y)
# Constraint 1 : Matmul outputs Y should be [email protected]
assert np.array_equal(Y, np.matmul(matmul.X, matmul.W.T))
# Constraint 2: Numerical gradient should be the same with numerical Jacobian
GN = matmul.gradient_numerical() # [dL/dX, dL/dW]
# DO NOT use matmul.function() as the objective function for numerical_jacobian().
# The state of the layer will be modified.
# LX = lambda x: matmul.objective(matmul.function(x))
def LX(x):
y = np.matmul(x, matmul.W.T)
# pylint: disable=not-callable
return matmul.objective(y)
EGNX = numerical_jacobian(LX,
matmul.X) # Numerical dL/dX including bias
EGNX = EGNX[::, 1::] # Remove bias for dL/dX
assert np.array_equal(GN[0], EGNX), \
"GN[0]\n%s\nEGNX=\n%s\n" % (GN[0], EGNX)
# DO NOT use matmul.function() as the objective function for numerical_jacobian().
# The state of the layer will be modified.
# LW = lambda w: matmul.objective(np.matmul(X, w.T))
def LW(w):
Y = np.matmul(matmul.X, w.T)
# pylint: disable=not-callable
return matmul.objective(Y)
EGNW = numerical_jacobian(LW,
matmul.W) # Numerical dL/dW including bias
assert np.array_equal(GN[1], EGNW) # No need to remove bias
# ================================================================================
# Layer backward path
# Calculate the analytical gradient dL/dX=matmul.gradient(dL/dY) with a dummy dL/dY.
# ================================================================================
dY = np.ones_like(Y)
dX = matmul.gradient(dY)
# Constraint 3: Matmul gradient dL/dX should be dL/dY @ W. Use a dummy dL/dY = 1.0.
expected_dX = np.matmul(dY, matmul.W)
expected_dX = expected_dX[::, 1:: # Omit bias
]
assert np.array_equal(dX, expected_dX)
# Constraint 5: Analytical gradient dL/dX close to the numerical gradient GN.
assert np.all(np.abs(dX - GN[0]) < GRADIENT_DIFF_ACCEPTANCE_VALUE), \
"dX need close to GN[0]. dX:\n%s\ndiff \n%s\n" % (dX, dX-GN[0])
# --------------------------------------------------------------------------------
# Gradient update.
# Run the gradient descent to update Wn+1 = Wn - lr * dL/dX.
# --------------------------------------------------------------------------------
# Python passes the reference to W, hence it is directly updated by the gradient-
# descent to avoid a temporary copy. Backup W before to compare before/after.
backup = copy.deepcopy(W)
# Gradient descent and returns analytical dL/dX, dL/dW
dS = matmul.update()
dW = dS[0]
# Constraint 6.: W has been updated by the gradient descent.
assert np.any(backup != matmul.W), "W has not been updated "
# Constraint 5: the numerical gradient (dL/dX, dL/dW) are closer to the analytical ones.
assert validate_against_expected_gradient(GN[0], dX), \
"dX=\n%s\nGN[0]=\n%sdiff=\n%s\n" % (dX, GN[0], (dX-GN[0]))
assert validate_against_expected_gradient(GN[1], dW), \
"dW=\n%s\nGN[1]=\n%sdiff=\n%s\n" % (dW, GN[1], (dW-GN[1]))
# Constraint 7: gradient descent progressing with the new objective L(Yn+1) < L(Yn)
# pylint: disable=not-callable
assert np.all(np.abs(objective(matmul.function(X)) < L))
profiler.disable()
profiler.print_stats(sort="cumtime")
def test_020_cross_entropy_log_loss_1d(caplog):
"""
Objective:
Test the categorical log loss values for P in 1 dimension.
Constraints:
1. The numerical gradient gn = (-t * logarithm(p+h) + t * logarithm(p-h)) / 2h.
2. The numerical gradient gn is within +/- u within the analytical g = -T/P.
P: Probabilities from softmax of shape (M,)
M: Number of nodes in the cross_entropy_log_loss layer.
T: Labels
Note:
log(P=1) -> 0
dlog(x)/dx = 1/x
"""
def f(P: np.ndarray, T: np.ndarray):
return np.sum(cross_entropy_log_loss(P, T))
# caplog.set_level(logging.DEBUG, logger=Logger.name)
h: TYPE_FLOAT = OFFSET_DELTA
u: TYPE_FLOAT = GRADIENT_DIFF_ACCEPTANCE_VALUE
# --------------------------------------------------------------------------------
# For (P, T): P[index] = True/1, OHE label T[index] = 1 where
# P=[0,0,0,...,1,...0], T = [0,0,0,...1,...0]. T[i] == 1
#
# Do not forget the Jacobian shape is (N,) and calculate each element.
# 1. For T=1, loss L = -log(Pi) = 0 and dL/dP=(1/Pi)= -1 is expected.
# 2. For T=0, Loss L = (-log(0+offset+h)-log(0+offset-h)) / 2h = 0 is expected.
# --------------------------------------------------------------------------------
M: TYPE_INT = np.random.randint(2, NUM_MAX_NODES)
index: TYPE_INT = TYPE_INT(np.random.randint(
0, M)) # Position of the true label in P
P1 = np.zeros(M, dtype=TYPE_FLOAT)
P1[index] = TYPE_FLOAT(1.0)
T1 = np.zeros(M, dtype=TYPE_LABEL)
T1[index] = TYPE_LABEL(1)
# Analytica correct gradient for P=1, T=1
AG = np.zeros_like(P1, dtype=TYPE_FLOAT)
AG[index] = TYPE_FLOAT(-1) # dL/dP = -1
EGN1 = np.zeros_like(P1, dtype=TYPE_FLOAT) # Expected numerical gradient
EGN1[index] = (-1 * logarithm(TYPE_FLOAT(1.0 + h)) + TYPE_FLOAT(1) *
logarithm(TYPE_FLOAT(1.0 - h))) / TYPE_FLOAT(2 * h)
assert np.all(np.abs(EGN1-AG) < u), \
"Expected EGN-1<%s but %s\nEGN=\n%s" % (u, (EGN1-AG), EGN1)
GN1 = numerical_jacobian(partial(f, T=T1), P1)
assert np.all(np.abs(GN1-AG) < u), \
"Expected GN-1<%s but %s\nGN=\n%s" % (u, (GN1-AG), GN1)
# The numerical gradient gn = (-t * logarithm(p+h) + t * logarithm(p-h)) / 2h
assert GN1.shape == EGN1.shape
assert np.all(np.abs(EGN1-GN1) < u), \
"Expected GN1==EGN1 but GN1-EGN1=\n%sP=\n%s\nT=%s\nEGN=\n%s\nGN=\n%s\n" \
% (np.abs(GN1-EGN1), P1, T1, EGN1, GN1)
# The numerical gradient gn is within +/- u within the analytical g = -T/P
G1 = np.zeros_like(P1, dtype=TYPE_FLOAT)
G1[T1 == 1] = -1 * (T1[index] / P1[index])
# G1[T1 != 0] = 0
check.equal(np.all(np.abs(G1 - GN1) < u), True,
"G1-GN1 %s\n" % np.abs(G1 - GN1))
# --------------------------------------------------------------------------------
# For (P, T): P[index] = np uniform(), index label T=index
# --------------------------------------------------------------------------------
for _ in range(NUM_MAX_TEST_TIMES):
M = np.random.randint(2, NUM_MAX_NODES) # M > 1
T2 = TYPE_LABEL(np.random.randint(0, M)) # location of the truth
P2 = np.zeros(M, dtype=TYPE_FLOAT)
while not (x := TYPE_FLOAT(
np.random.uniform(low=-BOUNDARY_SIGMOID,
high=BOUNDARY_SIGMOID))):
pass
p = softmax(x)
P2[T2] = p
# --------------------------------------------------------------------------------
# The Jacobian G shape is the same with P.shape.
# G:[0, 0, ...,g, 0, ...] where Gi is numerical gradient close to -1/(1+k).
# --------------------------------------------------------------------------------
N2 = np.zeros_like(P2, dtype=TYPE_FLOAT)
N2[T2] = TYPE_FLOAT(-1) * (logarithm(p + h) -
logarithm(p - h)) / TYPE_FLOAT(2 * h)
N2 = numerical_jacobian(partial(f, T=T2), P2)
# The numerical gradient gn = (-t * logarithm(p+h) + t * logarithm(p-h)) / 2h
assert N2.shape == N2.shape
assert np.all(np.abs(N2-N2) < u), \
f"Delta expected to be < {u} but \n{np.abs(N2-N2)}"
G2 = np.zeros_like(P2, dtype=TYPE_FLOAT)
G2[T2] = -1 / p
# The numerical gradient gn is within +/- u within the analytical g = -T/P
check.equal(np.all(np.abs(G2 - N2) < u), True,
"G2-N2 %s\n" % np.abs(G2 - N2))
def test_020_cross_entropy_log_loss_2d(caplog):
"""
Objective:
Test case for cross_entropy_log_loss(X, T) for X:shape(N,M), T:shape(N,)
Expected:
"""
def f(P: np.ndarray, T: np.ndarray):
"""Loss function"""
# For P.ndim==2 of shape (N, M), cross_entropy_log_loss() returns (N,).
# Each of which has the loss for P[n].
# If divided by P.shape[0] or N, the loss gets 1/N, which is wrong.
# This is not a gradient function but a loss function.
# return np.sum(cross_entropy_log_loss(P, T)) / P.shape[0]
return np.sum(cross_entropy_log_loss(P, T))
# caplog.set_level(logging.DEBUG, logger=Logger.name)
h: TYPE_FLOAT = OFFSET_DELTA
u: TYPE_FLOAT = GRADIENT_DIFF_ACCEPTANCE_VALUE
for _ in range(NUM_MAX_TEST_TIMES):
# --------------------------------------------------------------------------------
# [2D test case]
# P:(N, M) is probability matrix where Pnm = p, 0 <=n<N-1, 0<=m<M-1
# T:(N,) is index label where Tn=m is label as integer k.g. m=3 for 3rd label.
# Pnm = log(P[i][j])
# L = -log(p), -> dlog(P)/dP -> -1 / (p)
#
# Keep p value away from 0. As p gets close to 0, the log(p+/-h) gets large e.g
# -11.512925464970229, hence log(p+/-h) / 2h explodes.
# --------------------------------------------------------------------------------
while not (x := TYPE_FLOAT(
np.random.uniform(low=-BOUNDARY_SIGMOID,
high=BOUNDARY_SIGMOID))):
pass
p = softmax(x)
N = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M = np.random.randint(2, NUM_MAX_NODES)
# label index, not OHE
T = np.random.randint(0, M, N).astype(
TYPE_LABEL) # N rows of labels, max label value is M-1
P = np.zeros((N, M)).astype(TYPE_FLOAT)
P[range(N), # Set p at random row position
T] = p
E = np.zeros_like(P).astype(TYPE_FLOAT)
E[range(N), # Set p at random row position
T] = (TYPE_FLOAT(-1) * logarithm(p + h) +
TYPE_FLOAT(1) * logarithm(p - h)) / (TYPE_FLOAT(2) * h)
G = numerical_jacobian(partial(f, T=T), P)
assert E.shape == G.shape, \
f"Jacobian shape is expected to be {E.shape} but {G.shape}."
assert np.all(np.abs(E-G) < u), \
f"Delta expected to be < {u} but \n{np.abs(E-G)}"
A = np.zeros_like(P).astype(TYPE_FLOAT)
A[range(N), # Set p at random row position
T] = -1 / p
check.equal(np.all(np.abs(A - G) < u), True,
"A-G %s\n" % np.abs(A - G))
np.random.uniform(low=-BOUNDARY_SIGMOID,
high=BOUNDARY_SIGMOID))):
pass
p = softmax(x)
P3 = np.zeros(M, dtype=TYPE_FLOAT)
P3[index] = p
T3 = np.zeros(M).astype(TYPE_LABEL) # OHE index
T3[index] = TYPE_LABEL(1)
# --------------------------------------------------------------------------------
# The Jacobian G shape is the same with P.shape.
# --------------------------------------------------------------------------------
N3 = np.zeros_like(P3, dtype=TYPE_FLOAT)
N3[index] = TYPE_FLOAT(-1 * logarithm(p + h) +
1 * logarithm(p - h)) / TYPE_FLOAT(2 * h)
N3 = numerical_jacobian(partial(f, T=T3), P3)
assert N3.shape == N3.shape
assert np.all(np.abs(N3-N3) < u), \
f"Delta expected to be < {u} but \n{np.abs(N3-N3)}"
G3 = np.zeros_like(P3, dtype=TYPE_FLOAT)
G3[index] = -1 / p
check.equal(np.all(np.abs(G3 - N3) < u), True,
"G3-N3 %s\n" % np.abs(G3 - N3))
# --------------------------------------------------------------------------------
# [1D test case]
# For 1D OHE array P [0, 0, ..., 1, 0, ...] where Pi = 1.
# For 1D OHE array T [0, 0, ..., 0, 1, ...] where Tj = 1 and i != j
# sum(-t * logarithm(0)) -> log(offset)
# dlog(P)/dP -> -T / P
def disabled_test_040_objective_methods_1d_ohe():
"""
TODO: Disabled as need to redesign numerical_jacobian for 32 bit floating.
Objective:
Verify the forward path constraints:
1. Layer output L/loss is np.sum(cross_entropy_log_loss(sigmoid(X), T, f=logistic_log_loss))) / N.
2. gradient_numerical() == numerical Jacobian numerical_jacobian(O, X).
Verify the backward path constraints:
1. Analytical gradient G: gradient() == (P-1)/N
2. Analytical gradient G is close to GN: gradient_numerical().
Expected:
Initialization detects the access to the non-initialized parameters and fails.
For X.ndim > 0, the layer transform X into 2D so as to use the numpy tuple-
like indexing:
P[
(0,3),
(2,4)
]
Hence, the shape of GN, G are 2D.
"""
# --------------------------------------------------------------------------------
# Instantiate a CrossEntropyLogLoss layer
# --------------------------------------------------------------------------------
name = "test_040_objective_methods_1d_ohe"
N = 1
for _ in range(NUM_MAX_TEST_TIMES):
layer = CrossEntropyLogLoss(
name=name,
num_nodes=1,
log_loss_function=sigmoid_cross_entropy_log_loss,
log_level=logging.DEBUG)
# ================================================================================
# Layer forward path
# ================================================================================
X = TYPE_FLOAT(
np.random.uniform(low=-BOUNDARY_SIGMOID, high=BOUNDARY_SIGMOID))
T = TYPE_LABEL(np.random.randint(0, 2)) # OHE labels.
# log_loss function require (X, T) in X(N, M), and T(N, M) in OHE label format.
X, T = transform_X_T(X, T)
layer.T = T
# Expected analytical gradient dL/dX = (P-T)/N of shape (N,M)
A = sigmoid(X)
EG = ((A - T) / N).reshape(1, -1).astype(TYPE_FLOAT)
Logger.debug("%s: X is \n%s\nT is %s\nP is %s\nEG is %s\n", name, X, T,
A, EG)
# --------------------------------------------------------------------------------
# constraint: L/loss == np.sum(J) / N.
# J, P = sigmoid_cross_entropy_log_loss(X, T)
# --------------------------------------------------------------------------------
L = layer.function(X) # L is shape ()
J, P = sigmoid_cross_entropy_log_loss(X, T)
Z = np.array(np.sum(J), dtype=TYPE_FLOAT) / TYPE_FLOAT(N)
assert np.array_equal(L, Z), f"LogLoss output should be {L} but {Z}."
# --------------------------------------------------------------------------------
# constraint: gradient_numerical() == numerical Jacobian numerical_jacobian(O, X)
# Use a dummy layer for the objective function because using the "layer"
# updates the X, Y which can interfere the independence of the layer.
# --------------------------------------------------------------------------------
GN = layer.gradient_numerical() # [dL/dX] from the layer
# --------------------------------------------------------------------------------
# Cannot use CrossEntropyLogLoss.function() to simulate the objective function L.
# because it causes applying transform_X_T multiple times.
# Because internally transform_X_T(X, T) has transformed T into the index label
# in 1D with with length 1 by "T = T.reshape(-1)".
# Then providing X in 1D into "dummy.function(x)" re-run "transform_X_T(X, T)"
# again. The (X.ndim == T.ndim ==1) as an input and T must be OHE label for such
# combination and T.shape == P.shape must be true for OHE labels.
# However, T has been converted into the index format already by transform_X_T
# (applying transform_X_T multiple times) and (T.shape=(1,1), X.shape=(1, > 1)
# that violates the (X.shape == T.shape) constraint.
# --------------------------------------------------------------------------------
# dummy = CrossEntropyLogLoss(
# name="dummy",
# num_nodes=M,
# log_level=logging.DEBUG
# )
# dummy.T = T
# dummy.objective = objective
# dummy.function(X)
# --------------------------------------------------------------------------------
def objective(x):
j, p = sigmoid_cross_entropy_log_loss(x, T)
return np.array(np.sum(j) / N, dtype=TYPE_FLOAT)
EGN = numerical_jacobian(objective,
X).reshape(1, -1) # Expected numerical dL/dX
assert np.array_equal(GN[0], EGN), \
f"Layer gradient_numerical GN \n{GN} \nneeds to be \n{EGN}."
# ================================================================================
# Layer backward path
# ================================================================================
# --------------------------------------------------------------------------------
# constraint: Analytical gradient G: gradient() == (P-1)/N.
# --------------------------------------------------------------------------------
dY = TYPE_FLOAT(1)
G = layer.gradient(dY)
assert np.all(np.abs(G-EG) <= GRADIENT_DIFF_ACCEPTANCE_VALUE), \
f"Layer gradient dL/dX \n{G} \nneeds to be \n{EG}."
# --------------------------------------------------------------------------------
# constraint: Analytical gradient G is close to GN: gradient_numerical().
# --------------------------------------------------------------------------------
assert \
np.all(np.abs(G-GN[0]) <= GRADIENT_DIFF_ACCEPTANCE_VALUE) or \
np.all(np.abs(G-GN[0]) <= np.abs(GRADIENT_DIFF_ACCEPTANCE_RATIO * GN[0])), \
"dX is \n%s\nGN is \n%s\nG-GN is \n%s\n Ratio * GN[0] is \n%s.\n" \
% (G, GN[0], G-GN[0], GRADIENT_DIFF_ACCEPTANCE_RATIO * GN[0])