def factory(source: st.DataObject) -> None:
with type_vars():
with pure_functions():
with container_strategies(container_type, settings=settings):
source.draw(st.builds(law.definition))
def wrapper(shapes: hnp.BroadcastableShapes, data: st.DataObject):
self.index_to_arr_shapes.update((k, v) for k, v in zip(
sorted(self.missing_shapes), shapes.input_shapes))
# list of drawn arrays to feed to functions
arrs = data.draw(
st.tuples(*(self.arrays(i).map(Tensor)
for i in range(self.num_arrays))),
label="arrays",
)
arr_copies = tuple(copy(arr) for arr in arrs)
if callable(self.kwargs):
kwargs = data.draw(self.kwargs(*arrs), label="kwargs")
if not isinstance(kwargs, dict):
raise TypeError(
"`kwargs` was a search strategy. This needs to draw dictionaries,"
"instead drew: {}".format(kwargs))
else:
# The keyword args to be passed to `self.op`. If any provided argument is callable
# it is assumed to by a hypothesis search strategy, and all of the drawn arrays will
# be passed to the strategy, in order to draw a value for that keyword argument.
# Otherwise the provided value is used as-is.
kwargs = {
k: (data.draw(v(*arrs), label="kwarg: {}".format(k))
if callable(v) else v)
for k, v in self.kwargs.items()
}
if self.assumptions is not None:
assume(self.assumptions(*arrs, **kwargs))
for i, arr in enumerate(
arrs): # assure arrays don't contain forbidden values
for value in self.index_to_no_go.get(i, ()):
assume(np.all(arr != value))
# forward pass of the function
out = self.op(*arrs, **kwargs)
# gradient to be backpropped through this operation
grad = data.draw(
hnp.arrays(
shape=out.shape,
dtype=float,
elements=st.floats(-10, 10),
unique=True,
),
label="grad",
)
grad_copy = copy(grad) # keep a copy to check for later mutations
# compute analytic derivatives via mygrad-backprop
if any(out.shape != i.shape for i in arrs):
# Broadcasting occurred
# Must reduce `out` to scalar
# first multiply by `grad` to simulate non-trivial back-prop
(grad * out).sum().backward()
else:
out.backward(grad)
if not self.finite_difference:
# compute derivatives via numerical approximation of derivative
# using the complex-step method
numerical_grad = (numerical_gradient_full
if self.vary_each_element else
numerical_gradient)
else:
numerical_grad = finite_difference
grads_numerical = numerical_grad(self.true_func,
*(i.data for i in arrs),
back_grad=grad,
kwargs=kwargs)
# check that the analytic and numeric derivatives match
for n, (arr, d_num) in enumerate(zip(arrs, grads_numerical)):
assert_allclose(
arr.grad,
d_num,
**self.tolerances,
err_msg=
"arr-{}: mygrad derivative and numerical derivative do not match"
.format(n),
)
# check that none of the set derivatives is a view of `grad`
assert not np.shares_memory(
arr.grad,
grad), "arr-{}.grad stores a view of grad".format(n)
# check that none of the set derivatives are views of one another
for arr_i, arr_j in combinations(arrs, 2):
assert not np.shares_memory(
arr_i.grad, arr_j.grad
), "two input arrays were propagated views of the same gradient"
# verify that null_gradients works
out.null_gradients()
assert all(i.grad is None for i in arrs), "null_gradients failed"
# check if any of the input-arrays were mutated
for n, (arr, arr_copy) in enumerate(zip(arrs, arr_copies)):
assert_array_equal(
arr.data,
arr_copy.data,
err_msg="arr-{} was mutated during backward prop".format(
n),
)
# check if `grad` was mutated
assert_array_equal(
grad,
grad_copy,
err_msg="`grad` was mutated during backward prop")
def test_set_color(plotter: LivePlot, colors: dict, data: st.DataObject):
metric = data.draw(st.sampled_from(plotter.metrics), label="metric")
plotter.metric_colors = {metric: colors}
assert plotter.metric_colors[metric] == colors
def test_is_invalid_gradient(grad, is_invalid, data: st.DataObject):
if isinstance(grad, st.SearchStrategy):
grad = data.draw(grad, label="grad")
assert is_invalid_gradient(grad) is is_invalid, grad
def wrapper(shapes: hnp.BroadcastableShapes, constant,
data: st.DataObject):
self.index_to_arr_shapes.update((k, v) for k, v in zip(
sorted(self.missing_shapes), shapes.input_shapes))
# list of drawn arrays to feed to functions
arrs = data.draw(
st.tuples(*(self.arrays(i) for i in range(self.num_arrays))),
label="arrays",
)
# list of array-copies to check for mutation
arr_copies = tuple(copy(arr) for arr in arrs)
if callable(self.kwargs):
kwargs = data.draw(self.kwargs(*arrs))
if not isinstance(kwargs, dict):
raise TypeError(
"`kwargs` was a search strategy. This needs to draw dictionaries,"
"instead drew: {}".format(kwargs))
else:
# set or draw keyword args to be passed to functions
kwargs = {
k: (data.draw(v(*arrs), label="kwarg: {}".format(k))
if callable(v) else v)
for k, v in self.kwargs.items()
}
if self.assumptions is not None:
assume(self.assumptions(*arrs, **kwargs))
for i, arr in enumerate(
arrs): # assure arrays don't contain forbidden values
for value in self.index_to_no_go.get(i, ()):
assume(np.all(arr != value))
# execute mygrad and "true" functions. Compare outputs and check mygrad behavior
o = self.op(*(Tensor(i) for i in arrs),
**kwargs,
constant=constant)
tensor_out = o.data
true_out = self.true_func(*arrs, **kwargs)
assert isinstance(
o, Tensor
), "`mygrad_func` returned type {}, should return `mygrad.Tensor`".format(
type(o))
assert (
o.constant is constant
), "`mygrad_func` returned tensor.constant={}, should be constant={}".format(
o.constant, constant)
assert_allclose(
actual=tensor_out,
desired=true_out,
err_msg=
"`mygrad_func(x)` and `true_func(x)` produce different results",
**self.tolerances,
)
for n, (arr, arr_copy) in enumerate(zip(arrs, arr_copies)):
assert_array_equal(
arr,
arr_copy,
err_msg="arr-{} was mutated during forward prop".format(n),
)
def test_properties(data: DataObject,
coordinates_limits_type_pair: Tuple[ScalarsLimitsType,
ScalarsLimitsType],
sizes_pair: SizesPair, border_sizes_pair: SizesPair,
holes_sizes_pair: SizesPair,
hole_sizes_pair: SizesPair) -> None:
(x_coordinates_limits_type,
y_coordinates_limits_type) = coordinates_limits_type_pair
((x_coordinates, (min_x_value, max_x_value)),
x_type) = x_coordinates_limits_type
((y_coordinates, (min_y_value, max_y_value)),
y_type) = y_coordinates_limits_type
min_size, max_size = sizes_pair
min_border_size, max_border_size = border_sizes_pair
min_holes_size, max_holes_size = holes_sizes_pair
min_hole_size, max_hole_size = hole_sizes_pair
strategy = multipolygons(x_coordinates,
y_coordinates,
min_size=min_size,
max_size=max_size,
min_border_size=min_border_size,
max_border_size=max_border_size,
min_holes_size=min_holes_size,
max_holes_size=max_holes_size,
min_hole_size=min_hole_size,
max_hole_size=max_hole_size)
result = data.draw(strategy)
assert is_multipolygon(result)
assert multipolygon_has_valid_sizes(result,
min_size=min_size,
max_size=max_size,
min_border_size=min_border_size,
max_border_size=max_border_size,
min_holes_size=min_holes_size,
max_holes_size=max_holes_size,
min_hole_size=min_hole_size,
max_hole_size=max_hole_size)
assert multipolygon_has_coordinates_types(result,
x_type=x_type,
y_type=y_type)
assert multipolygon_has_coordinates_in_range(result,
min_x_value=min_x_value,
max_x_value=max_x_value,
min_y_value=min_y_value,
max_y_value=max_y_value)
assert is_multipolygon_strict(result)
assert all(
is_contour_non_self_intersecting(polygon.border) and all(
is_contour_non_self_intersecting(hole) for hole in polygon.holes)
for polygon in result.polygons)
assert contours_do_not_cross_or_overlap(
[polygon.border for polygon in result.polygons])
assert all(
contours_do_not_cross_or_overlap(polygon.holes)
for polygon in result.polygons)
assert all(
is_contour_counterclockwise(polygon.border) and all(
not is_contour_counterclockwise(hole) for hole in polygon.holes)
for polygon in result.polygons)
def wrapper(shapes: hnp.BroadcastableShapes, constant,
data: st.DataObject):
self.index_to_arr_shapes.update((k, v) for k, v in zip(
sorted(self.missing_shapes), shapes.input_shapes))
# list of drawn arrays to feed to functions
arrs = data.draw(
st.tuples(*(self.arrays(i) for i in range(self.num_arrays))),
label="arrays",
)
# list of array-copies to check for mutation
arr_copies = tuple(copy(arr) for arr in arrs)
if callable(self.kwargs):
kwargs = data.draw(self.kwargs(*arrs))
if not isinstance(kwargs, dict):
raise TypeError(
f"`kwargs` was a search strategy. This needs to draw dictionaries,"
f"instead drew: {kwargs}")
else:
# set or draw keyword args to be passed to functions
kwargs = {
k: (data.draw(v(
*arrs), label=f"kwarg: {f}") if callable(v) else v)
for k, v in self.kwargs.items()
}
if self.assumptions is not None:
assume(self.assumptions(*arrs, **kwargs))
for i, arr in enumerate(
arrs): # assure arrays don't contain forbidden values
for value in self.index_to_no_go.get(i, ()):
assume(np.all(arr != value))
if self.permit_0d_array_as_float:
# potentially cast a 0D array as a float
arrs = tuple(
arr.item() if arr.ndim == 0 and data.
draw(st.booleans(), label=f"arr-{n} to float") else arr
for n, arr in enumerate(arrs))
# execute mygrad and "true" functions. Compare outputs and check mygrad behavior
tensor_constants = data.draw(st.tuples(*[st.booleans()] *
len(arrs)),
label="tensor_constants")
o = self.op(
*(Tensor(i, constant=c)
for i, c in zip(arrs, tensor_constants)),
**kwargs,
constant=constant,
)
tensor_out = o.data
true_out = self.true_func(*arrs, **kwargs)
assert isinstance(
o, Tensor
), f"`mygrad_func` returned type {type(o)}, should return `mygrad.Tensor`"
assert o.constant is constant or bool(sum(tensor_constants)), (
f"`mygrad_func` returned tensor.constant={o.constant}, "
f"should be constant={constant or bool(sum(tensor_constants))}"
)
assert_allclose(
actual=tensor_out,
desired=true_out,
err_msg=
"`mygrad_func(x)` and `true_func(x)` produce different results",
**self.tolerances,
)
for n, (arr, arr_copy) in enumerate(zip(arrs, arr_copies)):
assert_array_equal(
arr,
arr_copy,
err_msg=f"arr-{n} was mutated during forward prop",
)
def test_gru_backward(
data: st.DataObject,
X: np.ndarray,
D: int,
bp_lim: bool,
dropout: bool,
U_constants: Tuple[bool, bool, bool],
W_constants: Tuple[bool, bool, bool],
b_constants: Tuple[bool, bool, bool],
X_constant: bool,
V_constant: bool,
):
tolerances = dict(atol=1e-5, rtol=1e-5)
T, N, C = X.shape
Wz, Wr, Wh = data.draw(
hnp.arrays(shape=(3, D, D), dtype=float, elements=st.floats(-10.0, 10.0)),
label="Wz, Wr, Wh",
)
Uz, Ur, Uh = data.draw(
hnp.arrays(shape=(3, C, D), dtype=float, elements=st.floats(-10.0, 10.0)),
label="Uz, Ur, Uh",
)
bz, br, bh = data.draw(
hnp.arrays(shape=(3, D), dtype=float, elements=st.floats(-10.0, 10.0)),
label="bz, br, bh",
)
V = data.draw(
hnp.arrays(shape=(D, C), dtype=float, elements=st.floats(-10.0, 10.0)),
label="V",
)
s0 = np.zeros((N, D), dtype=float)
X = Tensor(X, constant=X_constant)
X2 = X.__copy__()
Wz = Tensor(Wz, constant=W_constants[0])
Wz2 = Wz.__copy__()
Uz = Tensor(Uz, constant=U_constants[0])
Uz2 = Uz.__copy__()
bz = Tensor(bz, constant=b_constants[0])
bz2 = bz.__copy__()
Wr = Tensor(Wr, constant=W_constants[1])
Wr2 = Wr.__copy__()
Ur = Tensor(Ur, constant=U_constants[1])
Ur2 = Ur.__copy__()
br = Tensor(br, constant=b_constants[1])
br2 = br.__copy__()
Wh = Tensor(Wh, constant=W_constants[2])
Wh2 = Wh.__copy__()
Uh = Tensor(Uh, constant=U_constants[2])
Uh2 = Uh.__copy__()
bh = Tensor(bh, constant=b_constants[2])
bh2 = bh.__copy__()
V = Tensor(V, constant=V_constant)
V2 = V.__copy__()
s0 = Tensor(s0)
s2 = s0.__copy__()
# bp_lim = len(X) - 1 should behave the same as no bp-lim
s = gru(
X,
Uz,
Wz,
bz,
Ur,
Wr,
br,
Uh,
Wh,
bh,
dropout=dropout,
constant=False,
bp_lim=len(X) - 1 if bp_lim else None,
)
o = matmul(s[1:], V)
ls = o.sum()
ls.backward()
stt = s2
all_s = [s0.data]
ls2 = 0
if dropout:
Wz2d = s.creator._dropWz * Wz2
Wr2d = s.creator._dropWr * Wr2
Wh2d = s.creator._dropWh * Wh2
else:
Wz2d = Wz2
Wr2d = Wr2
Wh2d = Wh2
for n, x in enumerate(X2):
if not dropout:
z = sigmoid(matmul(x, Uz2) + matmul(stt, Wz2d) + bz2)
r = sigmoid(matmul(x, Ur2) + matmul(stt, Wr2d) + br2)
h = tanh(matmul(x, Uh2) + matmul((r * stt), Wh2d) + bh2)
else:
z = sigmoid(
(s.creator._dropUz[0] * matmul(x, Uz2)) + matmul(stt, Wz2d) + bz2
)
r = sigmoid(
(s.creator._dropUr[0] * matmul(x, Ur2)) + matmul(stt, Wr2d) + br2
)
h = tanh(
(s.creator._dropUh[0] * matmul(x, Uh2)) + matmul((r * stt), Wh2d) + bh2
)
stt = (1 - z) * h + z * stt
all_s.append(stt)
o = matmul(stt, V2)
ls2 += o.sum()
ls2.backward()
rec_s_grad = np.stack([i.grad for i in all_s[1:]])
if not s.constant:
assert_allclose(rec_s_grad, s.grad, **tolerances)
else:
assert s.grad is None
if not Wz.constant:
assert_allclose(Wz.grad, Wz2.grad, **tolerances)
else:
assert Wz.grad is None
if not Wr.constant:
assert_allclose(Wr.grad, Wr2.grad, **tolerances)
else:
assert Wr.grad is None
if not Wh.constant:
assert_allclose(Wh.grad, Wh2.grad, **tolerances)
else:
assert Wh.grad is None
if not Uz.constant:
assert_allclose(Uz.grad, Uz2.grad, **tolerances)
else:
assert Uz.grad is None
if not Ur.constant:
assert_allclose(Ur.grad, Ur2.grad, **tolerances)
else:
assert Ur.grad is None
if not Uh.constant:
assert_allclose(Uh.grad, Uh2.grad, **tolerances)
else:
assert Uh.grad is None
if not bz.constant:
assert_allclose(bz.grad, bz2.grad, **tolerances)
else:
assert bz.grad is None
if not br.constant:
assert_allclose(br.grad, br2.grad, **tolerances)
else:
assert br.grad is None
if not bh.constant:
assert_allclose(bh.grad, bh2.grad, **tolerances)
else:
assert bh.grad is None
if not V.constant:
assert_allclose(V.grad, V2.grad, **tolerances)
else:
assert V.grad is None
if not X.constant:
assert_allclose(X.grad, X2.grad, **tolerances)
else:
assert X.grad is None
ls.null_gradients()
ls2.null_gradients()
for x in [s, Wz, Wr, Wh, bz, br, bh, X, Uz, Ur, Uh, V]:
assert x.grad is None
def test_gru_fwd(X, D, dropout, data: st.DataObject):
T, N, C = X.shape
Wz, Wr, Wh = data.draw(
hnp.arrays(shape=(3, D, D), dtype=float, elements=st.floats(-10.0, 10.0)),
label="Wz, Wr, Wh",
)
Uz, Ur, Uh = data.draw(
hnp.arrays(shape=(3, C, D), dtype=float, elements=st.floats(-10.0, 10.0)),
label="Uz, Ur, Uh",
)
bz, br, bh = data.draw(
hnp.arrays(shape=(3, D), dtype=float, elements=st.floats(-10.0, 10.0)),
label="bz, br, bh",
)
V = data.draw(
hnp.arrays(shape=(D, C), dtype=float, elements=st.floats(-10.0, 10.0)),
label="V",
)
s0 = np.zeros((N, D), dtype=float)
X = Tensor(X)
X2 = X.__copy__()
Wz = Tensor(Wz)
Wz2 = Wz.__copy__()
Uz = Tensor(Uz)
Uz2 = Uz.__copy__()
bz = Tensor(bz)
bz2 = bz.__copy__()
Wr = Tensor(Wr)
Wr2 = Wr.__copy__()
Ur = Tensor(Ur)
Ur2 = Ur.__copy__()
br = Tensor(br)
br2 = br.__copy__()
Wh = Tensor(Wh)
Wh2 = Wh.__copy__()
Uh = Tensor(Uh)
Uh2 = Uh.__copy__()
bh = Tensor(bh)
bh2 = bh.__copy__()
V = Tensor(V)
V2 = V.__copy__()
s0 = Tensor(s0)
s2 = s0.__copy__()
s = gru(X, Uz, Wz, bz, Ur, Wr, br, Uh, Wh, bh, dropout=dropout, constant=True)
o = matmul(s[1:], V)
ls = o.sum()
assert s.constant is True
if dropout:
for d in [
s.creator._dropUr,
s.creator._dropUz,
s.creator._dropUh,
s.creator._dropWr,
s.creator._dropWz,
s.creator._dropWh,
]:
assert np.all(np.logical_or(d == 1 / (1 - dropout), d == 0))
stt = s2
all_s = [s0.data]
ls2 = 0
if dropout:
Wz2d = s.creator._dropWz * Wz2
Wr2d = s.creator._dropWr * Wr2
Wh2d = s.creator._dropWh * Wh2
else:
Wz2d = Wz2
Wr2d = Wr2
Wh2d = Wh2
for n, x in enumerate(X2):
if not dropout:
z = sigmoid(matmul(x, Uz2) + matmul(stt, Wz2d) + bz2)
r = sigmoid(matmul(x, Ur2) + matmul(stt, Wr2d) + br2)
h = tanh(matmul(x, Uh2) + matmul((r * stt), Wh2d) + bh2)
else:
z = sigmoid(
(s.creator._dropUz[0] * matmul(x, Uz2)) + matmul(stt, Wz2d) + bz2
)
r = sigmoid(
(s.creator._dropUr[0] * matmul(x, Ur2)) + matmul(stt, Wr2d) + br2
)
h = tanh(
(s.creator._dropUh[0] * matmul(x, Uh2)) + matmul((r * stt), Wh2d) + bh2
)
stt = (1 - z) * h + z * stt
all_s.append(stt)
o = matmul(stt, V2)
ls2 += o.sum()
tolerances = dict(atol=1e-5, rtol=1e-5)
rec_s_dat = np.stack([i.data for i in all_s])
assert_allclose(ls.data, ls2.data, **tolerances)
assert_allclose(rec_s_dat, s.data, **tolerances)
assert_allclose(Wz.data, Wz2.data, **tolerances)
assert_allclose(Wr.data, Wr2.data, **tolerances)
assert_allclose(Wh.data, Wh2.data, **tolerances)
assert_allclose(Uz.data, Uz2.data, **tolerances)
assert_allclose(Ur.data, Ur2.data, **tolerances)
assert_allclose(Uh.data, Uh2.data, **tolerances)
assert_allclose(bz.data, bz2.data, **tolerances)
assert_allclose(br.data, br2.data, **tolerances)
assert_allclose(bh.data, bh2.data, **tolerances)
assert_allclose(V.data, V2.data, **tolerances)
assert_allclose(X.data, X2.data, **tolerances)
ls.null_gradients()
for x in [s, Wz, Wr, Wh, bz, br, bh, X, Uz, Ur, Uh, V]:
assert x.grad is None
def test_tensors_shape(shape, data: st.DataObject):
tensor = data.draw(tensors(np.int8, shape=shape), label="tensor")
assert isinstance(tensor, Tensor)
assert tensor.shape == shape
assert tensor.grad is None
def test_tensors_dtype(dtype, data: st.DataObject):
tensor = data.draw(tensors(dtype=dtype, shape=(2, 3)), label="tensor")
assert isinstance(tensor, Tensor)
assert tensor.dtype == dtype
assert tensor.grad is None
def test_tensors_static_constant(constant: bool, data: st.DataObject):
tensor = data.draw(tensors(np.int8, (2, 3), constant=constant),
label="tensor")
assert isinstance(tensor, Tensor)
assert tensor.constant is constant
assert tensor.grad is None
def test_valid_shapes(arr: np.ndarray, data: st.DataObject):
newshape = data.draw(valid_shapes(arr.size), label="newshape")
arr.reshape(newshape)
def test_log_softmax_numerical_stability(x: np.ndarray, data: st.DataObject):
axis = data.draw(valid_axes(x.ndim), label="axis")
out = np.exp(logsoftmax(x, axis=axis).data)
assert np.all(np.logical_and(0 <= out, out <= 1)), out
assert_allclose(out.sum(axis=axis), 1.0)
def test_ranked_margin(shape: Tuple[int, ...], margin: float,
labels_as_tensor: bool, data: st.DataObject):
x1 = data.draw(
hnp.arrays(shape=shape, dtype=float, elements=st.floats(-1000, 1000)),
label="x1",
)
x2 = data.draw(
hnp.arrays(shape=shape, dtype=float, elements=st.floats(-1000, 1000)),
label="x2",
)
y = data.draw(
st.sampled_from((-1, 1))
| hnp.arrays(
shape=shape[:1],
dtype=hnp.integer_dtypes(),
elements=st.sampled_from((-1, 1)),
).map(lambda x: mg.Tensor(x) if labels_as_tensor else x),
label="y",
)
x1_copy = np.copy(x1)
x2_copy = np.copy(x2)
y_copy = y.copy() if isinstance(y, mg.Tensor) else np.copy(y)
x1_dum = mg.Tensor(x1)
x2_dum = mg.Tensor(x2)
x1_real = mg.Tensor(x1)
x2_real = mg.Tensor(x2)
loss_dum = simple_loss(x1_dum, x2_dum, y, margin)
loss_real = margin_ranking_loss(x1_real, x2_real, y, margin)
assert_allclose(actual=loss_real.data,
desired=loss_dum.data,
err_msg="losses don't match")
assert_array_equal(x1, x1_copy, err_msg="`x1` was mutated by forward")
assert_array_equal(x2, x2_copy, err_msg="`x2` was mutated by forward")
if isinstance(y, np.ndarray):
assert_array_equal(y, y_copy, err_msg="`y` was mutated by forward")
loss_dum.backward()
loss_real.backward()
assert_allclose(actual=x1_real.grad,
desired=x1_dum.grad,
err_msg="x1.grad doesn't match")
assert_allclose(actual=x2_real.grad,
desired=x2_dum.grad,
err_msg="x2.grad doesn't match")
assert_array_equal(x1, x1_copy, err_msg="`x1` was mutated by backward")
assert_array_equal(x2, x2_copy, err_msg="`x2` was mutated by backward")
if isinstance(y, mg.Tensor):
y = y.data
if isinstance(y, np.ndarray):
assert_array_equal(y, y_copy, err_msg="`y` was mutated by backward")
loss_real.null_gradients()
assert x1_real.grad is None
assert x2_real.grad is None
def test_properties(data: DataObject) -> None:
strategy = empty_geometries()
result = data.draw(strategy)
assert is_empty(result)
