def predict(network, x):
W1, W2, W3 = network['W1'], network['W2'], network['W3']
b1, b2, b3 = network['b1'], network['b2'], network['b3']
a1 = np.dot(x, W1) + b1
z1 = sigmoid(a1)
a2 = np.dot(z1, W2) + b2
z2 = sigmoid(a2)
a3 = np.dot(z2, W3) + b3
y = softmax(a3)
return y
def test_010_sigmoid():
"""Test Case for sigmoid
"""
u = ACTIVATION_DIFF_ACCEPTANCE_VALUE
assert sigmoid(np.array(TYPE_FLOAT(0),
dtype=TYPE_FLOAT)) == TYPE_FLOAT(0.5)
x = np.array([0.0, 0.6, 0., -0.5]).reshape((2, 2)).astype(TYPE_FLOAT)
t = np.array(
[0.5, 0.6456563062257954529091, 0.5,
0.3775406687981454353611]).reshape((2, 2)).astype(TYPE_FLOAT)
assert np.all(np.abs(t - sigmoid(x)) < u), \
f"delta (t-x) is expected < {u} but {x-t}"
def forward(self, x, t):
self.t = t
self.y = sigmoid(x)
self.loss = cross_entropy_error(np.c_[1 - self.y, self.y], self.t)
return self.loss
def predict(self, x):
w1, w2 = self.params['w1'], self.params['w2']
b1, b2 = self.params['b1'], self.params['b2']
a1 = np.dot(x, w1) + b1
z1 = sigmoid(a1)
a2 = np.dot(z1, w2) + b2
y = softmax(a2)
return y
def grad(self, x, t):
w1, w2 = self.params['w1'], self.params['w2']
b1, b2 = self.params['b1'], self.params['b2']
grads = {}
# forward
a1 = np.dot(x, w1) + b1
z1 = sigmoid(a1)
a2 = np.dot(z1, w2) + b2
y = softmax(a2)
# backward
dy = (y - t) / x.shape[0] # softmax with entropy loss's gradient, dL/dy
grads['w2'] = np.dot(z1.T, dy)
grads['b2'] = np.sum(dy, axis=0)
da1 = np.dot(dy, w2.T)
dz1 = (1.0 - sigmoid(a1)) * sigmoid(a1) * da1 # sigmoid's gradient
grads['w1'] = np.dot(x.T, dz1)
grads['b1'] = np.sum(dz1, axis=0)
return grads
def function(
self, X: Union[TYPE_FLOAT,
np.ndarray]) -> Union[TYPE_FLOAT, np.ndarray]:
X = np.array(X).reshape((1, -1)) if isinstance(X, TYPE_FLOAT) else X
assert X.shape[1] == self.M, \
f"Number of node X {X.shape[1] } does not match {self.M}."
self.X = X
if self._Y.size <= 0 or self.Y.shape[0] != self.N:
self._Y = np.empty(X.shape, dtype=TYPE_FLOAT)
self._Y = sigmoid(X, out=self._Y)
return self.Y
def test_020_numerical_jacobian_sigmoid(caplog):
"""Test Case for numerical gradient calculation
The domain of X is -BOUNDARY_SIGMOID < x < BOUNDARY_SIGMOID
Args:
u: Acceptable threshold value
"""
u: TYPE_FLOAT = GRADIENT_DIFF_ACCEPTANCE_VALUE
# y=sigmoid(x) -> dy/dx = y(1-y)
# 0.5 = sigmoid(0) -> dy/dx = 0.25
for _ in range(NUM_MAX_TEST_TIMES):
x = np.random.uniform(low=-BOUNDARY_SIGMOID,
high=BOUNDARY_SIGMOID,
size=1).astype(TYPE_FLOAT)
y = sigmoid(x)
analytical = np.multiply(y, (1 - y))
numerical = numerical_jacobian(sigmoid, x)
difference = np.abs(analytical - numerical)
acceptance = np.abs(analytical * GRADIENT_DIFF_ACCEPTANCE_RATIO)
assert np.all(difference < max(u, acceptance)), \
f"Needs difference < {max(u, acceptance)} but {difference}\nx is {x}"
def test_010_sigmoid_cross_entropy_log_loss_2d(caplog):
"""
Objective:
Test case for sigmoid_cross_entropy_log_loss(X, T) =
-( T * log(sigmoid(X)) + (1 -T) * log(1-sigmoid(X)) )
For the input X of shape (N,1) and T in index format of shape (N,1),
calculate the sigmoid log loss and verify the values are as expected.
Expected:
For Z = sigmoid(X) = 1 / (1 + exp(-X)) and T=[[1]]
Then -log(Z) should be almost same with sigmoid_cross_entropy_log_loss(X, T).
Almost because finite float precision always has rounding errors.
"""
# caplog.set_level(logging.DEBUG, logger=Logger.name)
u = REFORMULA_DIFF_ACCEPTANCE_VALUE
# --------------------------------------------------------------------------------
# [Test case 01]
# X:(N,M)=(1, 1). X=(x0) where x0=0 by which sigmoid(X) generates 0.5.
# Expected:
# sigmoid_cross_entropy_log_loss(X, T) == -log(0.5)
# --------------------------------------------------------------------------------
X = np.array([[TYPE_FLOAT(0.0)]])
T = np.array([TYPE_LABEL(1)])
X, T = transform_X_T(X, T)
E = -logarithm(np.array([TYPE_FLOAT(0.5)]))
J, P = sigmoid_cross_entropy_log_loss(X, T)
assert E.shape == J.shape
assert np.all(E == J), \
"Expected (E==J) but \n%s\nE=\n%s\nT=%s\nX=\n%s\nJ=\n%s\n" \
% (np.abs(E - J), E, T, X, J)
assert P == 0.5
# --------------------------------------------------------------------------------
# [Test case 02]
# For X:(N,1)
# --------------------------------------------------------------------------------
for _ in range(NUM_MAX_TEST_TIMES):
# X(N, M), and T(N,) in index label format
N = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M = 1 # always 1 for binary classification 0 or 1.
X = np.random.randn(N, M).astype(TYPE_FLOAT)
T = np.random.randint(0, M, N).astype(TYPE_LABEL)
X, T = transform_X_T(X, T)
Logger.debug("T is %s\nX is \n%s\n", T, X)
# ----------------------------------------------------------------------
# Expected value EJ for J and Z for P
# Note:
# To handle both index label format and OHE label format in the
# Loss layer(s), X and T are transformed into (N,1) shapes in
# transform_X_T(X, T) for logistic log loss.
# DO NOT squeeze Z nor P.
# ----------------------------------------------------------------------
Z = sigmoid(X)
EJ = np.squeeze(-(T * logarithm(Z) + TYPE_FLOAT(1-T) * logarithm(TYPE_FLOAT(1-Z))), axis=-1)
# **********************************************************************
# Constraint: Actual J should be close to EJ.
# **********************************************************************
J, P = sigmoid_cross_entropy_log_loss(X, T)
assert EJ.shape == J.shape
assert np.all(np.abs(EJ-J) < u), \
"Expected abs(EJ-J) < %s but \n%s\nEJ=\n%s\nT=%s\nX=\n%s\nJ=\n%s\n" \
% (u, np.abs(EJ-J), EJ, T, X, J)
# **********************************************************************
# Constraint: Actual P should be close to Z.
# **********************************************************************
assert np.all(np.abs(Z-P) < u), \
"EP \n%s\nP\n%s\nEP-P \n%s\n" % (Z, P, Z-P)
# ----------------------------------------------------------------------
# L = cross_entropy_log_loss(P, T) should be close to J
# ----------------------------------------------------------------------
L = cross_entropy_log_loss(P=Z, T=T, f=logistic_log_loss)
assert L.shape == J.shape
assert np.all(np.abs(L-J) < u), \
"Expected abs(L-J) < %s but \n%s\nL=\n%s\nT=%s\nX=\n%s\nJ=\n%s\n" \
% (u, np.abs(L-J), L, T, X, J)
def forward(self, x, **kwargs):
self.out = sigmoid(x)
return self.out
def disabled_test_040_objective_methods_2d_ohe(caplog):
"""
TODO: Disabled as need to redesign numerical_jacobian for 32 bit floating.
Objective:
Verify the forward path constraints:
1. Layer output L/loss is np.sum(sigmoid_cross_entropy_log_loss) / N.
2. gradient_numerical() == numerical Jacobian numerical_jacobian(O, X).
Verify the backward path constraints:
1. Analytical gradient G: gradient() == (P-1)/N
2. Analytical gradient G is close to GN: gradient_numerical().
"""
caplog.set_level(logging.DEBUG)
# --------------------------------------------------------------------------------
# Instantiate a CrossEntropyLogLoss layer
# --------------------------------------------------------------------------------
name = "test_040_objective_methods_2d_ohe"
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = 1 # node number is 1 for 0/1 binary classification.
layer = CrossEntropyLogLoss(
name=name,
num_nodes=M,
log_loss_function=sigmoid_cross_entropy_log_loss,
log_level=logging.DEBUG)
# ================================================================================
# Layer forward path
# ================================================================================
X = np.random.randn(N, M).astype(TYPE_FLOAT)
T = np.zeros_like(X, dtype=TYPE_LABEL) # OHE labels.
T[np.arange(N), np.random.randint(0, M, N)] = TYPE_LABEL(1)
# log_loss function require (X, T) in X(N, M), and T(N, M) in OHE label format.
X, T = transform_X_T(X, T)
layer.T = T
Logger.debug("%s: X is \n%s\nT is \n%s", name, X, T)
# --------------------------------------------------------------------------------
# Expected analytical gradient EG = (dX/dL) = (A-T)/N
# --------------------------------------------------------------------------------
A = sigmoid(X)
EG = ((A - T).astype(TYPE_FLOAT) / TYPE_FLOAT(N))
# --------------------------------------------------------------------------------
# Total loss Z = np.sum(J)/N
# Expected loss EL = sum((1-T)X + np.log(1 + np.exp(-X)))
# (J, P) = sigmoid_cross_entropy_log_loss(X, T) and J:shape(N,) where J:shape(N,)
# is loss for each input and P is activation by sigmoid(X).
# --------------------------------------------------------------------------------
L = layer.function(X)
J, P = sigmoid_cross_entropy_log_loss(X, T)
EL = np.array(np.sum((1 - T) * X + logarithm(1 + np.exp(-X))) / N,
dtype=TYPE_FLOAT)
# Constraint: A == P as they are sigmoid(X)
assert np.all(np.abs(A-P) < ACTIVATION_DIFF_ACCEPTANCE_VALUE), \
f"Need A==P==sigmoid(X) but A=\n{A}\n P=\n{P}\n(A-P)=\n{(A-P)}\n"
# Constraint: Log loss layer output L == sum(J) from the log loss function
Z = np.array(np.sum(J) / N, dtype=TYPE_FLOAT)
assert np.array_equal(L, Z), \
f"Need log loss layer output L == sum(J) but L=\n{L}\nZ=\n{Z}."
# Constraint: L/loss is close to expected loss EL.
assert np.all(np.abs(EL-L) < LOSS_DIFF_ACCEPTANCE_VALUE), \
"Need EL close to L but \nEL=\n{EL}\nL=\n{L}\n"
# --------------------------------------------------------------------------------
# constraint: gradient_numerical() == numerical_jacobian(objective, X)
# TODO: compare the diff to accommodate numerical errors.
# --------------------------------------------------------------------------------
GN = layer.gradient_numerical() # [dL/dX] from the layer
def objective(x):
"""Function to calculate the scalar loss L for cross entropy log loss"""
j, p = sigmoid_cross_entropy_log_loss(x, T)
return np.array(np.sum(j) / N, dtype=TYPE_FLOAT)
EGN = numerical_jacobian(objective, X) # Expected numerical dL/dX
assert np.array_equal(GN[0], EGN), \
f"GN[0]==EGN expected but GN[0] is \n%s\n EGN is \n%s\n" % (GN[0], EGN)
# ================================================================================
# Layer backward path
# ================================================================================
# constraint: Analytical gradient G: gradient() == (P-1)/N.
dY = TYPE_FLOAT(1)
G = layer.gradient(dY)
assert np.all(np.abs(G-EG) <= GRADIENT_DIFF_ACCEPTANCE_VALUE), \
f"Layer gradient dL/dX \n{G} \nneeds to be \n{EG}."
# constraint: Analytical gradient G is close to GN: gradient_numerical().
assert \
np.allclose(GN[0], G, atol=GRADIENT_DIFF_ACCEPTANCE_VALUE, rtol=GRADIENT_DIFF_ACCEPTANCE_RATIO), \
f"dX is \n{G}\nGN[0] is \n{GN[0]}\nRDiff is \n{G-GN[0]}.\n"
# constraint: Gradient g of the log loss layer needs -1 < g < 1
# abs(P-T) = abs(sigmoid(X)-T) cannot be > 1.
assert np.all(np.abs(G) < 1), \
f"Log loss layer gradient cannot be < -1 nor > 1 but\n{G}"
assert np.all(np.abs(GN[0]) < (1+GRADIENT_DIFF_ACCEPTANCE_RATIO)), \
f"Log loss layer gradient cannot be < -1 nor > 1 but\n{GN[0]}"
profiler.disable()
profiler.print_stats(sort="cumtime")
def disabled_test_040_objective_methods_1d_ohe():
"""
TODO: Disabled as need to redesign numerical_jacobian for 32 bit floating.
Objective:
Verify the forward path constraints:
1. Layer output L/loss is np.sum(cross_entropy_log_loss(sigmoid(X), T, f=logistic_log_loss))) / N.
2. gradient_numerical() == numerical Jacobian numerical_jacobian(O, X).
Verify the backward path constraints:
1. Analytical gradient G: gradient() == (P-1)/N
2. Analytical gradient G is close to GN: gradient_numerical().
Expected:
Initialization detects the access to the non-initialized parameters and fails.
For X.ndim > 0, the layer transform X into 2D so as to use the numpy tuple-
like indexing:
P[
(0,3),
(2,4)
]
Hence, the shape of GN, G are 2D.
"""
# --------------------------------------------------------------------------------
# Instantiate a CrossEntropyLogLoss layer
# --------------------------------------------------------------------------------
name = "test_040_objective_methods_1d_ohe"
N = 1
for _ in range(NUM_MAX_TEST_TIMES):
layer = CrossEntropyLogLoss(
name=name,
num_nodes=1,
log_loss_function=sigmoid_cross_entropy_log_loss,
log_level=logging.DEBUG)
# ================================================================================
# Layer forward path
# ================================================================================
X = TYPE_FLOAT(
np.random.uniform(low=-BOUNDARY_SIGMOID, high=BOUNDARY_SIGMOID))
T = TYPE_LABEL(np.random.randint(0, 2)) # OHE labels.
# log_loss function require (X, T) in X(N, M), and T(N, M) in OHE label format.
X, T = transform_X_T(X, T)
layer.T = T
# Expected analytical gradient dL/dX = (P-T)/N of shape (N,M)
A = sigmoid(X)
EG = ((A - T) / N).reshape(1, -1).astype(TYPE_FLOAT)
Logger.debug("%s: X is \n%s\nT is %s\nP is %s\nEG is %s\n", name, X, T,
A, EG)
# --------------------------------------------------------------------------------
# constraint: L/loss == np.sum(J) / N.
# J, P = sigmoid_cross_entropy_log_loss(X, T)
# --------------------------------------------------------------------------------
L = layer.function(X) # L is shape ()
J, P = sigmoid_cross_entropy_log_loss(X, T)
Z = np.array(np.sum(J), dtype=TYPE_FLOAT) / TYPE_FLOAT(N)
assert np.array_equal(L, Z), f"LogLoss output should be {L} but {Z}."
# --------------------------------------------------------------------------------
# constraint: gradient_numerical() == numerical Jacobian numerical_jacobian(O, X)
# Use a dummy layer for the objective function because using the "layer"
# updates the X, Y which can interfere the independence of the layer.
# --------------------------------------------------------------------------------
GN = layer.gradient_numerical() # [dL/dX] from the layer
# --------------------------------------------------------------------------------
# Cannot use CrossEntropyLogLoss.function() to simulate the objective function L.
# because it causes applying transform_X_T multiple times.
# Because internally transform_X_T(X, T) has transformed T into the index label
# in 1D with with length 1 by "T = T.reshape(-1)".
# Then providing X in 1D into "dummy.function(x)" re-run "transform_X_T(X, T)"
# again. The (X.ndim == T.ndim ==1) as an input and T must be OHE label for such
# combination and T.shape == P.shape must be true for OHE labels.
# However, T has been converted into the index format already by transform_X_T
# (applying transform_X_T multiple times) and (T.shape=(1,1), X.shape=(1, > 1)
# that violates the (X.shape == T.shape) constraint.
# --------------------------------------------------------------------------------
# dummy = CrossEntropyLogLoss(
# name="dummy",
# num_nodes=M,
# log_level=logging.DEBUG
# )
# dummy.T = T
# dummy.objective = objective
# dummy.function(X)
# --------------------------------------------------------------------------------
def objective(x):
j, p = sigmoid_cross_entropy_log_loss(x, T)
return np.array(np.sum(j) / N, dtype=TYPE_FLOAT)
EGN = numerical_jacobian(objective,
X).reshape(1, -1) # Expected numerical dL/dX
assert np.array_equal(GN[0], EGN), \
f"Layer gradient_numerical GN \n{GN} \nneeds to be \n{EGN}."
# ================================================================================
# Layer backward path
# ================================================================================
# --------------------------------------------------------------------------------
# constraint: Analytical gradient G: gradient() == (P-1)/N.
# --------------------------------------------------------------------------------
dY = TYPE_FLOAT(1)
G = layer.gradient(dY)
assert np.all(np.abs(G-EG) <= GRADIENT_DIFF_ACCEPTANCE_VALUE), \
f"Layer gradient dL/dX \n{G} \nneeds to be \n{EG}."
# --------------------------------------------------------------------------------
# constraint: Analytical gradient G is close to GN: gradient_numerical().
# --------------------------------------------------------------------------------
assert \
np.all(np.abs(G-GN[0]) <= GRADIENT_DIFF_ACCEPTANCE_VALUE) or \
np.all(np.abs(G-GN[0]) <= np.abs(GRADIENT_DIFF_ACCEPTANCE_RATIO * GN[0])), \
"dX is \n%s\nGN is \n%s\nG-GN is \n%s\n Ratio * GN[0] is \n%s.\n" \
% (G, GN[0], G-GN[0], GRADIENT_DIFF_ACCEPTANCE_RATIO * GN[0])
def test_040_objective_instantiation():
"""
Objective:
Verify the initialized layer instance provides its properties.
Expected:
* name, num_nodes, M, log_level are the same as initialized.
* X, T, dY, objective returns what is set.
* N, M property are provided after X is set.
* Y, P, L properties are provided after function(X).
* gradient(dL/dY) repeats dL/dY,
* gradient_numerical() returns 1
"""
name = "test_040_objective_instantiation"
for _ in range(NUM_MAX_TEST_TIMES):
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = 1
# For sigmoid log loss layer, the number of features N in X is the same with node number.
D: int = M
layer = CrossEntropyLogLoss(
name=name,
num_nodes=M,
log_loss_function=sigmoid_cross_entropy_log_loss,
log_level=logging.DEBUG)
# --------------------------------------------------------------------------------
# Properties
# --------------------------------------------------------------------------------
assert layer.name == name
assert layer.num_nodes == layer.M == M
layer._D = D
assert layer.D == D
X = np.random.randn(N, D).astype(TYPE_FLOAT)
layer.X = X
assert np.array_equal(layer.X, X)
assert layer.N == N == X.shape[0]
# For sigmoid log loss layer, the number of features N in X is the same with node number.
assert layer.M == X.shape[1]
layer._dX = X
assert np.array_equal(layer.dX, X)
T = np.random.randint(0, M, N).astype(TYPE_LABEL)
layer.T = T
assert np.array_equal(layer.T, T)
# layer.function() gives the total loss L in shape ().
# log_loss function require (X, T) in X(N, M), and T(N, M) in OHE label format.
X, T = transform_X_T(X, T)
L = layer.function(X)
J, P = sigmoid_cross_entropy_log_loss(X, T)
assert \
L.shape == () and np.allclose(L, (np.sum(J) / N).astype(TYPE_FLOAT)) and L == layer.Y, \
"After setting T, layer.function(X) generates the total loss L but %s" % L
# layer.function(X) sets layer.P to sigmoid_cross_entropy_log_loss(X, T)
# P is nearly equal with sigmoid(X)
assert \
np.array_equal(layer.P, P) and \
np.all(np.abs(layer.P - sigmoid(X)) < LOSS_DIFF_ACCEPTANCE_VALUE), \
"layer.function(X) needs to set P as sigmoid_cross_entropy_log_loss(X, T) " \
"which is close to sigmoid(X) but layer.P=\n%s\nP=\n%s\nsigmoid(X)=%s" \
% (layer.P, P, sigmoid(X))
# gradient of sigmoid cross entropy log loss layer is (P-T)/N
G = layer.gradient()
assert \
np.all(np.abs(G - ((P-T)/N)) < GRADIENT_DIFF_ACCEPTANCE_VALUE), \
"Gradient G needs (P-T)/N but G=\n%s\n(P-T)/N=\n%s\n" % (G, (P-T)/N)
layer.logger.debug("This is a pytest")
# pylint: disable=not-callable
assert \
layer.objective(np.array(1.0, dtype=TYPE_FLOAT)) \
== np.array(1.0, dtype=TYPE_FLOAT), \
"Objective function of the output/last layer is an identity function."
def train_binary_classifier(N: int,
D: int,
M: int,
X: np.ndarray,
T: np.ndarray,
W: np.ndarray,
log_loss_function: Callable,
optimizer: Optimizer,
num_epochs: int = 100,
test_numerical_gradient: bool = False,
log_level: int = logging.ERROR,
callback: Callable = None):
"""Test case for binary classification with matmul + log loss.
Args:
N: Batch size
D: Number of features
M: Number of nodes. 1 for sigmoid and 2 for softmax
X: train data
T: labels
W: weight
log_loss_function: cross entropy logg loss function
optimizer: Optimizer
num_epochs: Number of epochs to run
test_numerical_gradient: Flag if test the analytical gradient with the numerical one.
log_level: logging level
callback: callback function to invoke at the each epoch end.
"""
name = __name__
assert isinstance(T, np.ndarray) and np.issubdtype(
T.dtype, np.integer) and T.ndim == 1 and T.shape[0] == N
assert isinstance(
X, np.ndarray) and X.dtype == TYPE_FLOAT and X.ndim == 2 and X.shape[
0] == N and X.shape[1] == D
assert isinstance(
W, np.ndarray) and W.dtype == TYPE_FLOAT and W.ndim == 2 and W.shape[
0] == M and W.shape[1] == D + 1
assert num_epochs > 0 and N > 0 and D > 0
assert ((log_loss_function == sigmoid_cross_entropy_log_loss and M == 1) or
(log_loss_function == softmax_cross_entropy_log_loss and M >= 2))
# --------------------------------------------------------------------------------
# Instantiate a CrossEntropyLogLoss layer
# --------------------------------------------------------------------------------
loss = CrossEntropyLogLoss(name="loss",
num_nodes=M,
log_loss_function=log_loss_function,
log_level=log_level)
# --------------------------------------------------------------------------------
# Instantiate a Matmul layer
# --------------------------------------------------------------------------------
matmul = Matmul(name="matmul",
num_nodes=M,
W=W,
optimizer=optimizer,
log_level=log_level)
matmul.objective = loss.function
num_no_progress: int = 0 # how many time when loss L not decreased.
loss.T = T
history: List[np.ndarray] = [loss.function(matmul.function(X))]
for i in range(num_epochs):
# --------------------------------------------------------------------------------
# Layer forward path
# Calculate the matmul output Y=f(X), and get the loss L = objective(Y)
# Test the numerical gradient dL/dX=matmul.gradient_numerical().
# --------------------------------------------------------------------------------
Y = matmul.function(X)
L = loss.function(Y)
if not (i % 50): print(f"iteration {i} Loss {L}")
Logger.info("%s: iteration[%s]. Loss is [%s]", name, i, L)
# --------------------------------------------------------------------------------
# Constraint: 1. Objective/Loss L(Yn+1) after gradient descent < L(Yn)
# --------------------------------------------------------------------------------
if L >= history[-1] and (i % 20) == 1:
Logger.warning(
"Iteration [%i]: Loss[%s] has not improved from the previous [%s].",
i, L, history[-1])
if (num_no_progress := num_no_progress + 1) > 20:
Logger.error(
"The training has no progress more than %s times.",
num_no_progress)
# break
else:
num_no_progress = 0
history.append(L)
# --------------------------------------------------------------------------------
# Expected dL/dW.T = X.T @ dL/dY = X.T @ (P-T) / N, and dL/dX = dL/dY @ W
# P = sigmoid(X) or softmax(X)
# dL/dX = dL/dY * W is to use W BEFORE updating W.
# --------------------------------------------------------------------------------
P = None
if log_loss_function == sigmoid_cross_entropy_log_loss:
# P = sigmoid(np.matmul(X, W.T))
P = sigmoid(np.matmul(matmul.X, matmul.W.T))
P = P - T.reshape(-1, 1) # T(N,) -> T(N,1) to align with P(N,1)
assert P.shape == (
N, 1), "P.shape is %s T.shape is %s" % (P.shape, T.shape)
elif log_loss_function == softmax_cross_entropy_log_loss:
# matmul.X.shape is (N, D+1), matmul.W.T.shape is (D+1, M)
P = softmax(np.matmul(matmul.X, matmul.W.T)) # (N, M)
P[np.arange(N), T] -= 1
EDX = np.matmul(P / N, matmul.W) # (N,M) @ (M, D+1) -> (N, D+1)
EDX = EDX[::, 1:] # Hide the bias -> (N, D)
EDW = np.matmul(matmul.X.T,
P / N).T # ((D+1,N) @ (N, M)).T -> (M, D+1)
# --------------------------------------------------------------------------------
# Layer backward path
# 1. Calculate the analytical gradient dL/dX=matmul.gradient(dL/dY) with a dL/dY.
# 2. Gradient descent to update Wn+1 = Wn - lr * dL/dX.
# --------------------------------------------------------------------------------
before = copy.deepcopy(matmul.W)
dY = loss.gradient(TYPE_FLOAT(1))
dX = matmul.gradient(dY)
# gradient descent and get the analytical gradients dS=[dL/dX, dL/dW]
# dL/dX.shape = (N, D)
# dL/dW.shape = (M, D+1)
dS = matmul.update()
dW = dS[0]
# --------------------------------------------------------------------------------
# Constraint 1. W in the matmul has been updated by the gradient descent.
# --------------------------------------------------------------------------------
Logger.debug("W after is \n%s", matmul.W)
assert not np.array_equal(before, matmul.W), "W has not been updated."
if not validate_against_expected_gradient(EDX, dX):
Logger.warning("Expected dL/dX \n%s\nDiff\n%s", EDX, EDX - dX)
if not validate_against_expected_gradient(EDW, dW):
Logger.warning("Expected dL/dW \n%s\nDiff\n%s", EDW, EDW - dW)
if test_numerical_gradient:
# --------------------------------------------------------------------------------
# Numerical gradients gn=[dL/dX, dL/dW]
# dL/dX.shape = (N, D)
# dL/dW.shape = (M, D+1)
# --------------------------------------------------------------------------------
gn = matmul.gradient_numerical()
validate_against_numerical_gradient([dX] + dS, gn, Logger)
if callback:
# if W.shape[1] == 1 else callback(W=np.average(matmul.W, axis=0))
callback(W=matmul.W[0])
def forward(self, x):
self.out = sigmoid(x)
return self.out
def test_020_adapt_embedding_loss_adapter_gradient_to_succeed(caplog):
"""
Objective:
Verify the Adapter gradient method handles dY in shape (N, 1+SL)
Adapter.function(Y) returns
- For Y:(N, 1+SL), the return is in shape (N*(1+SL),1).
Log loss T is set to the same shape
Expected:
"""
caplog.set_level(logging.DEBUG)
name = "test_020_adapt_embedding_logistic_loss_function_multi_lines"
sentences = """
Verify the EventIndexing function can handle multi line sentences
the asbestos fiber <unk> is unusually <unk> once it enters the <unk>
with even brief exposures to it causing symptoms that show up decades later researchers said
"""
dictionary: EventIndexing = _instantiate_event_indexing()
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
# First validate the correct configuration, then change parameter one by one.
E = target_size = TYPE_INT(np.random.randint(1, 3))
C = context_size = TYPE_INT(2 * np.random.randint(1, 5))
SL = negative_sample_size = TYPE_INT(np.random.randint(1, 5))
event_vector_size: TYPE_INT = TYPE_INT(np.random.randint(5, 20))
W: TYPE_TENSOR = np.random.rand(dictionary.vocabulary_size,
event_vector_size)
loss, adapter, embedding, event_context = _instantiate(
name=name,
num_nodes=TYPE_INT(1),
target_size=target_size,
context_size=context_size,
negative_sample_size=negative_sample_size,
event_vector_size=event_vector_size,
dictionary=dictionary,
W=W,
log_level=logging.DEBUG,
)
# ================================================================================
# Forward path
# ================================================================================
# --------------------------------------------------------------------------------
# Event indexing
# --------------------------------------------------------------------------------
sequences = dictionary.function(sentences)
# --------------------------------------------------------------------------------
# Event context pairs
# --------------------------------------------------------------------------------
target_context_pairs = event_context.function(sequences)
# --------------------------------------------------------------------------------
# Embedding
# --------------------------------------------------------------------------------
Y = embedding.function(target_context_pairs)
N, _ = embedding.tensor_shape(Y)
batch_size = TYPE_FLOAT(N * (1 + SL))
# --------------------------------------------------------------------------------
# Adapter
# --------------------------------------------------------------------------------
Z = adapter.function(Y)
# --------------------------------------------------------------------------------
# Loss
# --------------------------------------------------------------------------------
L = loss.function(Z)
# ********************************************************************************
# Constraint:
# loss.T is set to the T by adapter.function()
# ********************************************************************************
T = np.zeros(shape=(N, (1 + SL)), dtype=TYPE_LABEL)
T[::, 0] = TYPE_LABEL(1)
assert embedding.all_equal(T.reshape(-1, 1), loss.T), \
"Expected T must equals loss.T. Expected\n%s\nLoss.T\n%s\n" % (T, loss.T)
# ********************************************************************************
# Constraint:
# Expected loss is sum(sigmoid_cross_entropy_log_loss(Y, T)) / (N*(1+SL))
# The batch size for the Log Loss is (N*(1+SL))
# ********************************************************************************
EJ, EP = sigmoid_cross_entropy_log_loss(X=Z, T=T.reshape(-1, 1))
EL = np.sum(EJ, dtype=TYPE_FLOAT) / batch_size
assert embedding.all_close(EL, L), \
"Expected EL=L but EL=\n%s\nL=\n%s\nDiff=\n%s\n" % (EL, L, (EL-L))
# ================================================================================
# Backward path
# ================================================================================
# ********************************************************************************
# Constraint:
# Expected dL/dY from the Log Loss is (P-T)/N
# ********************************************************************************
EDY = (sigmoid(Y) - T.astype(TYPE_FLOAT)) / batch_size
assert EDY.shape == Y.shape
dY = adapter.gradient(loss.gradient(TYPE_FLOAT(1)))
assert dY.shape == Y.shape
assert embedding.all_close(EDY, dY), \
"Expected EDY==dY. EDY=\n%s\nDiff\n%s\n" % (EDY, (EDY-dY))
profiler.disable()
profiler.print_stats(sort="cumtime")
