def test_operation_and_placeholder_gradients():
a_array = np.ones([10, 5])
b_array = np.ones([5, 10])
a_sp = sp.Variable(a_array)
b_sp = sp.Variable(b_array)
a = sp.Placeholder()
b = sp.Placeholder()
y = sp.Lazy(sp.matmul)(a, b)
a.assign_value(a_sp)
b.assign_value(b_sp)
result = y.run()
grads = sp.get_gradients(result)
sp_results = [result.array, grads[a_sp], grads[b_sp]]
def func(a, b):
return np.matmul(a, b)
y_np = func(a_array, b_array)
args = [a_array, b_array]
num_grads = numgrads(func, args, n=1, delta=1)
num_results = [y_np, num_grads[0], num_grads[1]]
for spval, numval in zip(sp_results, num_results):
error = rmse(spval, numval)
assert error < EPS, f"rmse = {error}"
def compare_results(args, sp_func, np_func, delta=1, eps=EPS):
"""Compares:
- SmallPebble function output against NumPy function output.
- SmallPebble gradient against numerical gradient.
Notes:
`delta` can be 1 for linear functions,
but should otherwise be a very small number.
`eps` may need to be adjusted, in the case of
inaccurate numerical approximations.
"""
# Compute SmallPebble results
args_sp = [sp.Variable(a) for a in args]
y_sp = sp_func(*args_sp)
grads_sp = sp.get_gradients(y_sp)
grads_sp = [grads_sp[var] for var in args_sp]
# Compute numerical results
y_np = np_func(*args)
grads_np = numgrads(np_func, args, n=1, delta=delta)
# Compare output values
error = rmse(y_sp.array, y_np)
if error > eps:
raise NumericalError("function output rmse:", error)
# Compare gradient values
for i, (spval, npval) in enumerate(zip(grads_sp, grads_np)):
if error > eps:
raise NumericalError(f"arg[{i}] gradient rmse:", error)
def test_maxpool2d():
np.random.seed(0)
images = np.random.random([4, 12, 14, 3])
images_sp = sp.Variable(images)
images_tf = tf.Variable(images)
for stride in range(1, 11, 3):
strides = [stride, stride]
for padding in ["SAME", "VALID"]:
strides = [stride, stride]
result_sp = sp.maxpool2d(images_sp, 2, 2, padding, strides)
gradients_sp = sp.get_gradients(result_sp)
grad_images_sp = gradients_sp[images_sp]
with tf.GradientTape() as tape:
result_tf = tf.nn.max_pool2d(images_tf, [1, 2, 2, 1], strides, padding)
grad_images_tf = tape.gradient(result_tf, [images_tf])[0]
# Compare results:
results_error = rmse(result_sp.array, result_tf)
assert results_error < EPS, f"Results error = {results_error}"
# Compare gradients:
imgrad_error = np.mean(np.abs(grad_images_sp - grad_images_tf))
assert imgrad_error < EPS, f"Gradient error = {imgrad_error}"
def test_sgd_step():
a = sp.Variable(np.random.random([3, 5]))
b = sp.Variable(np.random.random([5, 3]))
y_true = sp.Variable(np.ones([3, 3]))
losses = []
for _ in range(100):
y_pred = sp.matmul(a, b)
loss = sp.sum(sp.square(y_true - y_pred))
losses.append(loss.array)
gradients = sp.get_gradients(loss)
sp.sgd_step([a, b], gradients)
assert np.all(np.diff(losses) < 0), 'Not converging.'
def test_conv2d_grads():
"Compare TensorFlow derivatives with SmallPebble."
images, kernels = generate_images_and_kernels([5, 16, 12, 2], [2, 4, 2, 3])
images_sp = sp.Variable(images)
kernels_sp = sp.Variable(kernels)
images_tf = tf.Variable(images)
kernels_tf = tf.Variable(kernels)
for stride in range(1, 11, 3):
strides = [stride, stride]
for padding in ["SAME", "VALID"]:
print("\n\n")
print(stride)
print(padding)
# Calculate convolution and gradients with revdiff:
result_sp = sp.conv2d(images_sp,
kernels_sp,
padding=padding,
strides=strides)
gradients_sp = sp.get_gradients(result_sp)
grad_images_sp = gradients_sp[images_sp]
grad_kernels_sp = gradients_sp[kernels_sp]
# Calculate convolution and gradients with tensorflow:
with tf.GradientTape() as tape:
result_tf = tf.nn.conv2d(images_tf,
kernels_tf,
padding=padding,
strides=strides)
grad_images_tf, grad_kernels_tf = tape.gradient(
result_tf, [images_tf, kernels_tf])
# Compare the gradients:
imgrad_error = np.mean(np.abs(grad_images_sp - grad_images_tf))
assert imgrad_error < EPS, f"Image gradient error = {imgrad_error}"
kerngrad_error = np.mean(np.abs(grad_kernels_sp - grad_kernels_tf))
assert kerngrad_error < EPS, f"Kernel gradient error = {kerngrad_error}"