def test_020_std_function_method_to_fail():
"""
Objective:
Verify the _layer class instance function validates invalid inputs
Expected:
Layer method fails.
"""
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
# For which works on statistics on per-feature basis,
# no sense if M = 1 or N = 1.
M: int = np.random.randint(2, NUM_MAX_NODES)
momentum = TYPE_FLOAT(0.85)
try:
_layer = _instance(name=name, num_nodes=M, momentum=momentum)
_layer.function(int(1))
raise RuntimeError("Invoke _layer.function(int(1)) must fail.")
except AssertionError:
pass
try:
_layer = _instance(name=name, num_nodes=M, momentum=momentum)
_layer.gradient(int(1))
raise RuntimeError("Invoke _layer.gradient(int(1)) must fail.")
except AssertionError:
pass
def test_020_fss_method_function_multi_invocations_to_succeed():
"""
Objective:
Verify the layer class instance function method
Expected:
Layer method calculate expected values.
"""
def objective(X: np.ndarray) -> Union[float, np.ndarray]:
"""Dummy objective function"""
return np.sum(X, dtype=TYPE_FLOAT)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
numexpr_enabled = bool(np.random.randint(0, 2))
# For BN which works on statistics on per-feature basis,
# no sense if M = 1 or N = 1.
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
X = np.random.randn(N, M).astype(TYPE_FLOAT)
# ********************************************************************************
# Constraint:
# layer needs to reallocate X related storages upon X.shape[0] change.
# ********************************************************************************
_layer = _instance(name=name, num_nodes=M, log_level=logging.DEBUG)
_layer.objective = objective
for i in range(np.random.randint(1, 100)):
_layer.function(
X,
numexpr_enabled=numexpr_enabled,
)
while True:
Z = np.random.randn(np.random.randint(1, NUM_MAX_BATCH_SIZE), M)
if Z.shape[0] != N:
Z = Z.astype(TYPE_FLOAT)
break
_layer.function(
Z,
numexpr_enabled=numexpr_enabled,
)
# ********************************************************************************
# Constraint: gamma, beta should match those of Z
# ********************************************************************************
_validate_storage_allocation(_layer, Z)
profiler.disable()
profiler.print_stats(sort="cumtime")
def test_020_std_instance_properties_access_to_succeed():
"""
Objective:
Verify the _layer class instance has initialized its properties.
Expected:
Layer parameter access to succeed
"""
def objective(X: TYPE_TENSOR):
"""Dummy objective function"""
return np.sum(X)
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
M: int = np.random.randint(1, NUM_MAX_NODES)
_layer = Standardization(
name=name,
num_nodes=M,
log_level=logging.DEBUG
)
_layer.objective = objective
assert _layer.name == name
assert _layer.num_nodes == M
assert \
_layer.U.dtype == TYPE_FLOAT and \
_layer.U.shape == (M,)
assert \
_layer.dU.dtype == TYPE_FLOAT and \
_layer.dU.size == M
assert \
_layer.dV.dtype == TYPE_FLOAT and \
_layer.dV.size == M
assert \
_layer.SD.dtype == TYPE_FLOAT and \
_layer.SD.shape == (M,)
assert \
_layer.norm.dtype == TYPE_FLOAT and \
_layer.norm.shape == (M,)
assert \
_layer.RU.dtype == TYPE_FLOAT and \
_layer.RU.shape == (M,)
assert \
_layer.RSD.dtype == TYPE_FLOAT and \
_layer.RSD.shape == (M,)
assert _layer.objective == objective
def test_020_fss_method_gradient_descent():
"""
Objective:
Verify the gradient descent
Expected:
The objective decrease with the descents.
"""
def objective(X: np.ndarray) -> Union[float, np.ndarray]:
"""Dummy objective function"""
return np.sum(X)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
numexpr_enabled = bool(np.random.randint(0, 2))
N: int = np.random.randint(2, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
X = np.random.randn(N, M).astype(TYPE_FLOAT)
_layer = _instance(name=name, num_nodes=M, log_level=logging.DEBUG)
_layer.objective = objective
u = GRADIENT_DIFF_ACCEPTANCE_VALUE
for _ in range(np.random.randint(1, 10)):
dout = np.random.uniform(-1, 1, size=X.shape).astype(TYPE_FLOAT)
Y = _layer.function(
X,
numexpr_enabled=numexpr_enabled,
)
# pylint: disable=not-callable
L = _layer.objective(Y)
G = _layer.gradient(
dY=dout,
numexpr_enabled=numexpr_enabled,
)
dGamma, dBeta = _layer.update()
# ********************************************************************************
# Constraint: expected gradients match with actual
# ********************************************************************************
expected_dGamma = np.sum(dout * _layer.X, axis=0, dtype=TYPE_FLOAT)
expected_dBeta = np.sum(dout, axis=0, dtype=TYPE_FLOAT)
assert np.allclose(expected_dGamma, dGamma, atol=u), \
"Need dGamma\n%s\nbut\n%s\ndiff=\n%s\n" \
% (expected_dGamma, dGamma, expected_dGamma-dGamma)
assert np.allclose(expected_dBeta, dBeta, atol=u), \
"Need dBeta\n%s\nbut\n%s\ndiff=\n%s\n" \
% (expected_dBeta, dBeta, expected_dBeta-dBeta)
def test_020_event_context_instance_properties(caplog):
"""
Objective:
Verify the layer class validates the parameters have been initialized before accessed.
Expected:
Initialization detects the access to the non-initialized parameters and fails.
"""
caplog.set_level(logging.DEBUG)
name = "test_020_event_context_instance_properties"
msg = "Accessing uninitialized property of the layer must fail."
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
stride: TYPE_INT = TYPE_INT(np.random.randint(1, 100))
event_size: TYPE_INT = TYPE_INT(np.random.randint(1, 100))
window_size: TYPE_INT = 2 * stride + event_size
name = random_string(np.random.randint(1, 10))
event_context = _must_succeed(name=name,
num_nodes=TYPE_INT(1),
window_size=window_size,
event_size=event_size,
msg=msg)
# --------------------------------------------------------------------------------
# To pass
# --------------------------------------------------------------------------------
try:
if not event_context.name == name:
raise RuntimeError("event_context.name == name should be true")
except AssertionError as e:
raise RuntimeError(
"Access to name should be allowed as already initialized."
) from e
try:
if not isinstance(event_context.logger, logging.Logger):
raise RuntimeError(
"isinstance(event_context.logger, logging.Logger) should be true"
)
except AssertionError as e:
raise RuntimeError(
"Access to logger should be allowed as already initialized."
) from e
assert event_context.window_size == window_size
assert event_context.event_size == event_size
profiler.disable()
profiler.print_stats(sort="cumtime")
def test_020_fss_method_function_to_succeed():
"""
Objective:
Verify the layer class instance function method
Expected:
Layer method calculate expected values.
"""
def objective(X: np.ndarray) -> Union[float, np.ndarray]:
"""Dummy objective function"""
return np.sum(X, dtype=TYPE_FLOAT)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
numexpr_enabled = bool(np.random.randint(0, 2))
numba_enabled = bool(np.random.randint(0, 2))
# For BN which works on statistics on per-feature basis,
# no sense if M = 1 or N = 1.
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
X = np.random.randn(N, M).astype(TYPE_FLOAT)
_layer = FeatureScaleShift(name=name,
num_nodes=M,
log_level=logging.DEBUG)
_layer.objective = objective
_layer.function(X)
# ********************************************************************************
# Constraint:
# _layer.N provides the latest X.shape[0]
# X related arrays should have its storage allocated and has the X.shape.
# * dX
# ********************************************************************************
assert _layer.N == X.shape[0]
assert \
_layer.dX.dtype == TYPE_FLOAT and \
_layer.dX.shape == (N, M)
profiler.disable()
profiler.print_stats(sort="cumtime")
def test_020_fss_instance_properties_access_to_succeed():
"""
Objective:
Verify the layer class instance has initialized its properties.
Expected:
Layer parameter access to succeed
"""
def objective(X: np.ndarray) -> Union[float, np.ndarray]:
"""Dummy objective function"""
return np.sum(X)
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
M: int = np.random.randint(1, NUM_MAX_NODES)
_layer = FeatureScaleShift(name=name,
num_nodes=M,
log_level=logging.DEBUG)
_layer.objective = objective
assert _layer.name == name
assert _layer.num_nodes == M
assert \
_layer.gamma.dtype == TYPE_FLOAT and \
_layer.gamma.shape == (M,) and \
np.all(_layer.gamma == np.ones(M, dtype=TYPE_FLOAT))
assert \
_layer.dGamma.dtype == TYPE_FLOAT and \
_layer.dGamma.shape == (M,)
assert \
_layer.beta.dtype == TYPE_FLOAT and \
_layer.beta.shape == (M,) and \
np.all(_layer.beta == np.zeros(M, dtype=TYPE_FLOAT))
assert \
_layer.dBeta.dtype == TYPE_FLOAT and \
_layer.dBeta.shape == (M,)
assert _layer.objective == objective
def test_020_fss_method_predict():
"""
Objective:
Verify the prediction function
Expected:
The objective
"""
# pylint: disable=not-callable
def objective(X: np.ndarray) -> Union[float, np.ndarray]:
"""Dummy objective function"""
return np.sum(X, dtype=TYPE_FLOAT)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
numexpr_enabled = bool(np.random.randint(0, 2))
# For BN which works on statistics on per-feature basis,
# no sense if M = 1 or N = 1.
N: int = np.random.randint(2, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
X = np.random.randn(N, M).astype(TYPE_FLOAT)
_layer = _instance(name=name, num_nodes=M, log_level=logging.DEBUG)
_layer.objective = objective
Y = _layer.function(
X,
numexpr_enabled=numexpr_enabled,
)
# ********************************************************************************
# Constraint: With only 1 invocation, predict should be the same with Y.
# ********************************************************************************
assert np.allclose(Y,
_layer.predict(X),
atol=TYPE_FLOAT(1e-9),
rtol=TYPE_FLOAT(0))
def test_020_std_method_function_multi_invocations_to_succeed():
"""
Objective:
Verify the _layer class instance function method
Expected:
Layer method calculate expected values.
"""
def objective(x: TYPE_TENSOR):
"""Dummy objective function"""
return np.sum(x)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
numexpr_enabled = bool(np.random.randint(0, 2))
# For which works on statistics on per-feature basis,
# no sense if M = 1 or N = 1.
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
X = np.random.randn(N, M).astype(TYPE_FLOAT)
momentum = TYPE_FLOAT(np.random.uniform(0.7, 0.99))
if np.random.uniform() < 0.5:
eps = TYPE_FLOAT(np.random.uniform(1e-12, 1e-10))
else:
eps = TYPE_FLOAT(0.0)
# ********************************************************************************
# Constraint:
# _layer needs to reallocate X related storages upon X.shape[0] change.
# ********************************************************************************
_layer: Standardization = \
_instance(name=name, num_nodes=M, momentum=momentum, eps=eps, log_level=logging.DEBUG)
_layer.objective = objective
for i in range(np.random.randint(1, 100)):
_layer.function(
X,
numexpr_enabled=numexpr_enabled,
)
total_rows_processed = _layer.total_rows_processed
ru = _layer.RU
rsd = _layer.RSD
while True:
Z = np.random.randn(np.random.randint(1, NUM_MAX_BATCH_SIZE), M)
if Z.shape[0] != N:
Z = Z.astype(TYPE_FLOAT)
break
_layer.function(
Z,
numexpr_enabled=numexpr_enabled,
)
# ********************************************************************************
# Constraint: Properties of Y, U, Xmd, SD should match those of Z
# ********************************************************************************
_validate_storage_allocation(_layer, Z)
_validate_layer_values(_layer, Z, eps=eps)
# ********************************************************************************
# Constraint: Statistics is updated with Z
# ********************************************************************************
assert _layer.total_rows_processed == total_rows_processed + Z.shape[0]
_validate_layer_running_statistics(
_layer=_layer, previous_ru=ru, previous_rsd=rsd, X=Z, eps=eps
)
profiler.disable()
profiler.print_stats(sort="cumtime")
def test_020_std_method_function_to_succeed():
"""
Objective:
Verify the _layer class instance function method
Expected:
Layer method calculate expected values.
"""
def objective(x: TYPE_TENSOR):
"""Dummy objective function"""
return np.sum(x, dtype=TYPE_FLOAT)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
numexpr_enabled = bool(np.random.randint(0, 2))
numba_enabled = bool(np.random.randint(0, 2))
# For which works on statistics on per-feature basis,
# no sense if M = 1 or N = 1.
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
X = np.random.randn(N, M).astype(TYPE_FLOAT)
momentum = TYPE_FLOAT(np.random.uniform(0.7, 0.99))
eps = TYPE_FLOAT(np.random.uniform(1e-12, 1e-10)) \
if np.random.uniform() < 0.5 else TYPE_FLOAT(0)
_layer: Standardization = \
_instance(name=name, num_nodes=M, momentum=momentum, eps=eps)
_layer.objective = objective
# ********************************************************************************
# Constraint: total_rows_processed = times_of_invocations * N
# ********************************************************************************
assert _layer.total_rows_processed == 0
ru = _layer.RU
rsd = _layer.RSD
_layer.function(
X,
numexpr_enabled=numexpr_enabled,
)
_validate_layer_values(_layer, X, eps=eps)
_validate_layer_running_statistics(
_layer=_layer, previous_ru=ru, previous_rsd=rsd, X=X, eps=eps
)
# ********************************************************************************
# Constraint:
# _layer.N provides the latest X.shape[0]
# X related arrays should have its storage allocated and has the X.shape.
# * dX
# * dXmd01
# * dXmd02
# ********************************************************************************
assert _layer.N == X.shape[0]
assert \
_layer.dX.dtype == TYPE_FLOAT and \
_layer.dX.shape == (N, M)
assert \
_layer.dXmd01.dtype == TYPE_FLOAT and \
_layer.dXmd01.shape == (N, M)
assert \
_layer.dXmd02.dtype == TYPE_FLOAT and \
_layer.dXmd02.shape == (N, M)
assert _layer.total_rows_processed == N
# ********************************************************************************
# Constraint: total_rows_processed = times_of_invocations * N
# ********************************************************************************
for i in range(np.random.randint(1, 100)):
_layer.function(
X,
numexpr_enabled=numexpr_enabled,
)
assert _layer.total_rows_processed == TYPE_INT(N * (i + 2))
profiler.disable()
profiler.print_stats(sort="cumtime")
def test_020_std_instance_properties_access_to_fail():
"""
Objective:
Verify the _layer class validates the parameters have been initialized before accessed.
Expected:
Initialization detects the access to the non-initialized parameters and fails.
"""
msg = "Accessing uninitialized property of the _layer must fail."
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
M: int = np.random.randint(1, NUM_MAX_NODES)
_layer = Standardization(
name=name,
num_nodes=M,
log_level=logging.DEBUG
)
# --------------------------------------------------------------------------------
# To pass
# --------------------------------------------------------------------------------
try:
if not _layer.name == name:
raise RuntimeError("_layer.name == name should be true")
except AssertionError:
raise RuntimeError("Access to name should be allowed as already initialized.")
try:
if not _layer.M == M:
raise RuntimeError("_layer.M == M should be true")
except AssertionError:
raise RuntimeError("Access to M should be allowed as already initialized.")
try:
if not isinstance(_layer.logger, logging.Logger):
raise RuntimeError("isinstance(_layer.logger, logging.Logger) should be true")
except AssertionError:
raise RuntimeError("Access to logger should be allowed as already initialized.")
# --------------------------------------------------------------------------------
# To fail
# --------------------------------------------------------------------------------
try:
print(_layer.X)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.N)
raise RuntimeError(msg)
except AssertionError:
pass
try:
_layer.X = int(1)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.dX)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.Xmd)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.dXmd01)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.dXmd02)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.Y)
raise RuntimeError(msg)
except AssertionError:
pass
try:
_layer._Y = int(1)
print(_layer.Y)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.dY)
raise RuntimeError(msg)
except AssertionError:
pass
try:
_layer._dY = int(1)
print(_layer.dY)
raise RuntimeError(msg)
except AssertionError:
pass
try:
# pylint: disable=not-callable
_layer.objective(np.array(1.0, dtype=TYPE_FLOAT))
raise RuntimeError(msg)
except AssertionError:
pass
def test_020_matmul_save_load():
"""
Objective:
Verify the load/save methods.
Constraints:
1. Be able to save the layer state.
2. Be able to load the layer state and the state is same with the layer state S.
"""
name = "test_020_matmul_save_load"
def objective(X: np.ndarray) -> Union[float, np.ndarray]:
"""Dummy objective function to calculate the loss L"""
return np.sum(X)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(1, NUM_MAX_NODES)
D: int = np.random.randint(1, NUM_MAX_FEATURES)
name = "test_020_matmul_methods"
matmul = _instantiate(name=name,
num_nodes=M,
num_features=D,
objective=objective)
X = _generate_X(N, D)
Y = matmul.function(X)
matmul.gradient(Y)
matmul.update()
backup_S = copy.deepcopy(matmul.S)
backup_W = copy.deepcopy(matmul.W)
pathname = os.path.sep + os.path.sep.join(
["tmp", name + random_string(12) + ".pkl"])
# ********************************************************************************
# Constraint:
# load must fail with the saved file being deleted.
# Make sure load() is loading the path
# ********************************************************************************
matmul.save(pathname)
path = pathlib.Path(pathname)
path.unlink()
try:
msg = "load must fail with the saved file being deleted."
matmul.load(pathname)
raise AssertionError(msg)
except RuntimeError as e:
pass
# ********************************************************************************
# Constraint:
# load restore the state before save
# ********************************************************************************
matmul.save(pathname)
matmul._W = np.zeros(shape=matmul.W.shape, dtype=TYPE_FLOAT)
matmul.load(pathname)
assert np.array_equal(backup_W, matmul.W), \
"expected \n%s\n actual \n%s\n" % (backup_W, matmul.W)
path = pathlib.Path(pathname)
path.unlink()
Y = matmul.function(X)
matmul.gradient(Y)
matmul.update()
profiler.disable()
profiler.print_stats(sort="cumtime")
def test_020_bn_method_function_validate_with_frederik_kratzert():
"""
Objective:
Verify the layer class instance function method calculates expected values
Expected:
Layer method calculate expected values.
"""
def objective(x: np.ndarray):
"""Dummy objective function"""
return np.sum(x)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
numexpr_enabled = bool(np.random.randint(0, 2))
numba_enabled = bool(np.random.randint(0, 2))
# For BN which works on statistics on per-feature basis,
# no sense if M = 1 or N = 1.
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
X = np.random.rand(N, M).astype(TYPE_FLOAT)
momentum = TYPE_FLOAT(np.random.uniform(0.7, 0.99))
if np.random.uniform() < 0.5:
eps = TYPE_FLOAT(np.random.uniform(1e-12, 1e-10))
else:
eps = TYPE_FLOAT(0.0)
layer = BatchNormalization(name=name,
num_nodes=M,
momentum=momentum,
eps=eps,
log_level=logging.DEBUG)
layer.objective = objective
u = GRADIENT_DIFF_ACCEPTANCE_VALUE
out, cache = batchnorm_forward(x=X,
gamma=layer.gamma,
beta=layer.beta,
eps=eps)
xhat, gamma, xmu, norm, sd, var, eps = cache
Y = layer.function(X,
numexpr_enabled=numexpr_enabled,
numba_enabled=numba_enabled)
# ********************************************************************************
# Constraint: Xsd, X-U, Xmd, SD should match those of frederik_kratzert
# ********************************************************************************
assert np.allclose(Y, out, atol=u), \
"Y=\n%s\nout=\n%s\ndiff=\n%s\n" \
% (Y, out, (out-Y))
assert np.allclose(layer.Xmd, xmu, atol=u), \
"Xmd=\n%s\nxmu=\n%s\ndiff=\n%s\n" \
% (layer.Xmd, xmu, (xmu-layer.Xmd))
assert np.allclose(layer.SD, sd, atol=u), \
"SD=\n%s\nsd=\n%s\ndiff=\n%s\n" \
% (layer.SD, sd, (sd-layer.SD))
assert np.allclose(layer.Xstd, xhat, atol=u), \
"Xstd=\n%s\nxhat=\n%s\ndiff=\n%s\n" \
% (layer.Xstd, xhat, (xhat-layer.Xstd))
profiler.disable()
profiler.print_stats(sort="cumtime")
def test_020_bn_instance_properties_access_to_succeed():
"""
Objective:
Verify the layer class instance has initialized its properties.
Expected:
Layer parameter access to succeed
"""
def objective(x: np.ndarray):
"""Dummy objective function"""
return np.sum(x)
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
M: int = np.random.randint(1, NUM_MAX_NODES)
layer = BatchNormalization(name=name,
num_nodes=M,
log_level=logging.DEBUG)
layer.objective = objective
assert layer.name == name
assert layer.num_nodes == M
assert \
layer.gamma.dtype == TYPE_FLOAT and \
layer.gamma.shape == (M,) and \
np.all(layer.gamma == np.ones(M, dtype=TYPE_FLOAT))
assert \
layer.dGamma.dtype == TYPE_FLOAT and \
layer.dGamma.shape == (M,)
assert \
layer.beta.dtype == TYPE_FLOAT and \
layer.beta.shape == (M,) and \
np.all(layer.beta == np.zeros(M, dtype=TYPE_FLOAT))
assert \
layer.dBeta.dtype == TYPE_FLOAT and \
layer.dBeta.shape == (M,)
assert \
layer.U.dtype == TYPE_FLOAT and \
layer.U.shape == (M,)
assert \
layer.dU.dtype == TYPE_FLOAT and \
layer.dU.size == M
assert \
layer.dV.dtype == TYPE_FLOAT and \
layer.dV.size == M
assert \
layer.SD.dtype == TYPE_FLOAT and \
layer.SD.shape == (M,)
assert \
layer.norm.dtype == TYPE_FLOAT and \
layer.norm.shape == (M,)
assert \
layer.RU.dtype == TYPE_FLOAT and \
layer.RU.shape == (M,)
assert \
layer.RSD.dtype == TYPE_FLOAT and \
layer.RSD.shape == (M,)
assert layer.objective == objective
def test_020_std_method_predict():
"""
Objective:
Verify the prediction function
Expected:
The objective
"""
def objective(x: TYPE_TENSOR):
"""Dummy objective function"""
return np.sum(x)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
numexpr_enabled = bool(np.random.randint(0, 2))
# For which works on statistics on per-feature basis,
# no sense if M = 1 or N = 1.
N: int = np.random.randint(2, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
X = np.random.randn(N, M).astype(TYPE_FLOAT)
momentum = TYPE_FLOAT(np.random.uniform(0.7, 0.99))
if np.random.uniform() < 0.5:
eps = TYPE_FLOAT(np.random.uniform(1e-12, 1e-8))
else:
eps = TYPE_FLOAT(0.0)
_layer: Standardization = \
_instance(
name=name,
num_nodes=M,
momentum=momentum,
eps=eps,
log_level=logging.DEBUG
)
_layer.objective = objective
Y = _layer.function(
X,
numexpr_enabled=numexpr_enabled
)
# ********************************************************************************
# Constraint: With only 1 invocation, predict should be the same with Y.
# RU = momentum * RU + (1 - momentum) * U
# After the 1st invocation, RU==U. Then momentum * U + (1 - momentum) * U -> U
# ********************************************************************************
assert np.allclose(Y, _layer.predict(X), atol=TYPE_FLOAT(1e-9), rtol=TYPE_FLOAT(0))
# ********************************************************************************
# Constraint: At 2nd invocation, predict should be the same with
#
# ********************************************************************************
Z = np.random.randn(N, M).astype(TYPE_FLOAT)
standardized, mean, sd, deviation = standardize(Z, eps=eps, keepdims=False)
expected_RU = _layer.RU * momentum + mean * (TYPE_FLOAT(1)-momentum)
expected_RSD = _layer.RSD * momentum + sd * (TYPE_FLOAT(1)-momentum)
_layer.function(
Z,
numexpr_enabled=numexpr_enabled
)
assert np.allclose(_layer.RU, expected_RU, atol=TYPE_FLOAT(1e-10), rtol=TYPE_FLOAT(0))
assert np.allclose(_layer.RSD, expected_RSD, atol=TYPE_FLOAT(1e-10), rtol=TYPE_FLOAT(0))
def test_020_matmul_instance_properties():
"""
Objective:
Verify the layer class validates the parameters have been initialized before accessed.
Expected:
Initialization detects the access to the non-initialized parameters and fails.
"""
msg = "Accessing uninitialized property of the layer must fail."
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
M: int = np.random.randint(1, NUM_MAX_NODES)
D: int = np.random.randint(1, NUM_MAX_FEATURES)
matmul = Matmul(name=name,
num_nodes=M,
W=weights.uniform(M, D + 1),
log_level=logging.DEBUG)
# --------------------------------------------------------------------------------
# To pass
# --------------------------------------------------------------------------------
try:
if not matmul.name == name:
raise RuntimeError("matmul.name == name should be true")
except AssertionError as e:
raise RuntimeError(
"Access to name should be allowed as already initialized."
) from e
try:
if not matmul.M == M:
raise RuntimeError("matmul.M == M should be true")
except AssertionError as e:
raise RuntimeError(
"Access to M should be allowed as already initialized.") from e
try:
if not isinstance(matmul.logger, logging.Logger):
raise RuntimeError(
"isinstance(matmul.logger, logging.Logger) should be true")
except AssertionError as e:
raise RuntimeError(
"Access to logger should be allowed as already initialized."
) from e
try:
a = matmul.D
except AssertionError:
raise RuntimeError(
"Access to D should be allowed as already initialized.")
try:
matmul.W is not None
except AssertionError:
raise RuntimeError(
"Access to W should be allowed as already initialized.")
try:
matmul.optimizer is not None
except AssertionError:
raise RuntimeError(
"Access to optimizer should be allowed as already initialized."
)
# --------------------------------------------------------------------------------
# To fail
# --------------------------------------------------------------------------------
try:
print(matmul.X)
raise RuntimeError(msg)
except AssertionError:
pass
try:
matmul.X = int(1)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(matmul.dX)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(matmul.dW)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(matmul.Y)
raise RuntimeError(msg)
except AssertionError:
pass
try:
matmul._Y = int(1)
print(matmul.Y)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(matmul.dY)
raise RuntimeError(msg)
except AssertionError:
pass
try:
matmul._dY = int(1)
print(matmul.dY)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(matmul.T)
raise RuntimeError(msg)
except AssertionError:
pass
try:
matmul.T = float(1)
raise RuntimeError(msg)
except AssertionError:
pass
try:
# pylint: disable=not-callable
matmul.objective(np.array(1.0, dtype=TYPE_FLOAT))
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(matmul.N)
raise RuntimeError(msg)
except AssertionError:
pass
assert matmul.name == name
assert matmul.num_nodes == M
try:
matmul = Matmul(name=name,
num_nodes=M,
W=weights.xavier(M, D + 1),
log_level=logging.DEBUG)
matmul.function(int(1))
raise RuntimeError("Invoke matmul.function(int(1)) must fail.")
except AssertionError:
pass
try:
matmul = Matmul(name=name,
num_nodes=M,
W=weights.xavier(M, D + 1),
log_level=logging.DEBUG)
matmul.gradient(int(1))
raise RuntimeError("Invoke matmul.gradient(int(1)) must fail.")
except AssertionError:
pass
def test_020_bn_method_gradient_validate_with_frederik_kratzert():
"""
Objective:
Verify the layer class instance gradient method calculates expected values
Expected:
Layer method calculate expected values.
"""
if TYPE_FLOAT == np.float32:
# TODO:
"Need to investigate/redesign for 32 bit floating"
return
def objective(x: np.ndarray):
"""Dummy objective function"""
return np.sum(x)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
numexpr_enabled = bool(np.random.randint(0, 2))
numba_enabled = bool(np.random.randint(0, 2))
# For BN which works on statistics on per-feature basis,
# no sense if M = 1 or N = 1.
N: int = np.random.randint(1, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
X = np.random.rand(N, M).astype(TYPE_FLOAT)
momentum = TYPE_FLOAT(np.random.uniform(0.7, 0.99))
if np.random.uniform() < 0.5:
eps = TYPE_FLOAT(np.random.uniform(1e-12, 1e-10))
else:
eps = TYPE_FLOAT(0.0)
_layer = BatchNormalization(name=name,
num_nodes=M,
momentum=momentum,
eps=eps,
log_level=logging.DEBUG)
_layer.objective = objective
u = GRADIENT_DIFF_ACCEPTANCE_VALUE
dout = np.ones(X.shape)
# --------------------------------------------------------------------------------
# Benchmark (frederik_kratzert)
# --------------------------------------------------------------------------------
out, cache = batchnorm_forward(x=X,
gamma=_layer.gamma,
beta=_layer.beta,
eps=eps)
xhat, gamma, xmu, norm, sd, var, eps = cache
dx, dgamma, dbeta, dxhat, dvar, dxmu2, dxmu1, dmu = batchnorm_backward(
dout, cache)
# ********************************************************************************
# Constraint: layer gradients should match those of frederik_kratzert
# ********************************************************************************
_layer.function(X,
numexpr_enabled=numexpr_enabled,
numba_enabled=numba_enabled)
_layer.gradient(dY=_layer.tensor_cast(dout, TYPE_FLOAT),
numexpr_enabled=numexpr_enabled,
numba_enabled=numba_enabled)
assert np.allclose(_layer.dGamma, dgamma, atol=u), \
"dGamma=\n%s\ndgamma=\n%s\ndiff=\n%s\n" \
% (_layer.dGamma, dgamma, (dgamma-_layer.dGamma))
assert np.allclose(_layer.dBeta, dbeta, atol=u), \
"dBeta=\n%s\ndbeta=\n%s\ndiff=\n%s\n" \
% (_layer.dBeta, dbeta, (dbeta - _layer.dBeta))
assert np.allclose(_layer.dXstd, dxhat, atol=u), \
"dXstd=\n%s\ndxhat=\n%s\ndiff=\n%s\n" \
% (_layer.dXstd, dxhat, (dxhat - _layer.dXstd))
assert np.allclose(_layer.dV, dvar, atol=u), \
"dV=\n%s\ndvar=\n%s\ndiff=\n%s\n" \
% (_layer.dV, dvar, (dvar - _layer.dV))
assert np.allclose(_layer.dXmd01, dxmu2, atol=u), \
"dXmd01=\n%s\ndxmu2=\n%s\ndiff=\n%s\n" \
% (_layer.dXmd01, dxmu2, (dxmu2 - _layer.dXmd01))
assert np.allclose(_layer.dXmd02, dxmu1, atol=u), \
"dXmd02=\n%s\ndxmu1=\n%s\ndiff=\n%s\n" \
% (_layer.dXmd02, dxmu1, (dxmu1 - _layer.dXmd02))
assert np.allclose(_layer.dU, dmu, atol=u), \
"dU=\n%s\ndmu=\n%s\ndiff=\n%s\n" \
% (_layer.dU, dmu, (dmu - _layer.dU))
assert np.allclose(_layer.dX, dx, atol=u), \
"dX=\n%s\ndx=\n%s\ndiff=\n%s\n" \
% (_layer.dX, dx, (dx - _layer.dX))
profiler.disable()
profiler.print_stats(sort="cumtime")
def test_030_objective_instance_properties():
"""
Objective:
Verify the layer class validates the parameters have been initialized before accessed.
Expected:
Initialization detects the access to the non-initialized parameters and fails.
"""
msg = "Accessing uninitialized property of the layer must fail."
name = random_string(np.random.randint(1, 10))
for _ in range(NUM_MAX_TEST_TIMES):
M: int = np.random.randint(2, NUM_MAX_NODES)
_layer = layer.CrossEntropyLogLoss(name=name,
num_nodes=M,
log_level=logging.DEBUG)
# --------------------------------------------------------------------------------
# To pass
# --------------------------------------------------------------------------------
try:
if not _layer.name == name:
raise RuntimeError("layer.name == name should be true")
except AssertionError:
raise RuntimeError(
"Access to name should be allowed as already initialized.")
try:
if not _layer.M == M:
raise RuntimeError("layer.M == M should be true")
except AssertionError:
raise RuntimeError(
"Access to M should be allowed as already initialized.")
try:
if not isinstance(_layer.logger, logging.Logger):
raise RuntimeError(
"isinstance(layer.logger, logging.Logger) should be true")
except AssertionError:
raise RuntimeError(
"Access to logger should be allowed as already initialized.")
# --------------------------------------------------------------------------------
# To fail
# --------------------------------------------------------------------------------
try:
print(_layer.X)
raise RuntimeError(msg)
except AssertionError:
pass
try:
_layer.X = int(1)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.N)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.dX)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.Y)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.P)
raise RuntimeError(msg)
except AssertionError:
pass
try:
_layer._Y = int(1)
print(_layer.Y)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.dY)
raise RuntimeError(msg)
except AssertionError:
pass
try:
_layer._dY = int(1)
print(_layer.dY)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.T)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.L)
raise RuntimeError(msg)
except AssertionError:
pass
try:
print(_layer.J)
raise RuntimeError(msg)
except AssertionError:
pass
try:
_layer.T = float(1)
raise RuntimeError(msg)
except AssertionError:
pass
try:
_layer.function(int(1))
raise RuntimeError("Invoke layer.function(int(1)) must fail.")
except AssertionError:
pass
try:
_layer.function(TYPE_FLOAT(1.0))
_layer.gradient(int(1))
raise RuntimeError("Invoke layer.gradient(int(1)) must fail.")
except AssertionError:
pass
del _layer
def test_020_bn_method_gradient_descent():
"""
Objective:
Verify the gradient descent
Expected:
The objective decrease with the descents.
"""
if TYPE_FLOAT == np.float32:
# TODO:
"Need to investigate/redesign for 32 bit floating"
return
def objective(x: np.ndarray):
"""Dummy objective function"""
return np.sum(x)
profiler = cProfile.Profile()
profiler.enable()
for _ in range(NUM_MAX_TEST_TIMES):
name = random_string(np.random.randint(1, 10))
numexpr_enabled = bool(np.random.randint(0, 2))
# For BN which works on statistics on per-feature basis,
# no sense if M = 1 or N = 1.
N: int = np.random.randint(2, NUM_MAX_BATCH_SIZE)
M: int = np.random.randint(2, NUM_MAX_NODES)
# DO not use np.random.rand as test fails for 32 bit float
X = np.random.rand(N, M).astype(TYPE_FLOAT)
momentum = TYPE_FLOAT(np.random.uniform(0.7, 0.99))
if np.random.uniform() < 0.5:
eps = TYPE_FLOAT(np.random.uniform(1e-12, 1e-10))
else:
eps = TYPE_FLOAT(0.0)
layer = BatchNormalization(name=name,
num_nodes=M,
momentum=momentum,
eps=eps,
log_level=logging.DEBUG)
layer.objective = objective
u = GRADIENT_DIFF_ACCEPTANCE_VALUE
for _ in range(np.random.randint(1, 10)):
dout = np.random.uniform(-1, 1, size=X.shape).astype(TYPE_FLOAT)
Y = layer.function(
X,
numexpr_enabled=numexpr_enabled,
)
# pylint: disable=not-callable
layer.objective(Y)
layer.gradient(
dY=dout,
numexpr_enabled=numexpr_enabled,
)
dGamma, dBeta = layer.update()
# ********************************************************************************
# Constraint: expected gradients match with actual
# ********************************************************************************
expected_dGamma = np.sum(dout * layer.Xstd, axis=0)
expected_dBeta = np.sum(dout, axis=0)
assert np.allclose(expected_dGamma, dGamma, atol=u), \
"Need dGamma\n%s\nbut\n%s\ndiff=\n%s\n" \
% (expected_dGamma, dGamma, expected_dGamma-dGamma)
assert np.allclose(expected_dBeta, dBeta, atol=u), \
"Need dBeta\n%s\nbut\n%s\ndiff=\n%s\n" \
% (expected_dBeta, dBeta, expected_dBeta-dBeta)
