def test_ranked_margin(shape, margin, data):
x1 = data.draw(
hnp.arrays(shape=shape, dtype=float, elements=st.floats(-1000, 1000)),
label="x1",
)
x2 = data.draw(
hnp.arrays(shape=shape, dtype=float, elements=st.floats(-1000, 1000)),
label="x2",
)
y = data.draw(
st.sampled_from((-1, 1))
| hnp.arrays(
shape=shape[:1],
dtype=hnp.integer_dtypes(),
elements=st.sampled_from((-1, 1)),
),
label="y",
)
x1_copy = np.copy(x1)
x2_copy = np.copy(x2)
y_copy = np.copy(y)
x1_dum = mg.Tensor(x1)
x2_dum = mg.Tensor(x2)
x1_real = mg.Tensor(x1)
x2_real = mg.Tensor(x2)
loss_dum = simple_loss(x1_dum, x2_dum, y, margin)
loss_real = margin_ranking_loss(x1_real, x2_real, y, margin)
assert_allclose(actual=loss_real.data,
desired=loss_dum.data,
err_msg="losses don't match")
assert_array_equal(x1, x1_copy, err_msg="`x1` was mutated by forward")
assert_array_equal(x2, x2_copy, err_msg="`x2` was mutated by forward")
if isinstance(y, np.ndarray):
assert_array_equal(y, y_copy, err_msg="`y` was mutated by forward")
loss_dum.backward()
loss_real.backward()
assert_allclose(actual=x1_real.grad,
desired=x1_dum.grad,
err_msg="x1.grad doesn't match")
assert_allclose(actual=x2_real.grad,
desired=x2_dum.grad,
err_msg="x2.grad doesn't match")
assert_array_equal(x1, x1_copy, err_msg="`x1` was mutated by backward")
assert_array_equal(x2, x2_copy, err_msg="`x2` was mutated by backward")
if isinstance(y, np.ndarray):
assert_array_equal(y, y_copy, err_msg="`y` was mutated by backward")
loss_real.null_gradients()
assert x1_real.grad is None
assert x2_real.grad is None
def __init__(self, in_dim, out_dim, initializer=Uniform, bias=True):
self.in_dim = in_dim
self.out_dim = out_dim
self.weights = mg.Tensor(np.random.rand(out_dim, in_dim))
if bias:
self.bias = mg.Tensor(np.random.rand(out_dim))
else:
self.bias = None
def test_average_scl():
x = mg.Tensor(np.array([1, 2, 3, 4]))
weights = mg.Tensor(np.array([1, 2, 3, 4]))
avg, scl = average(x, weights=weights, returned=True)
expected_avg, expected_scl = np.average(x.data,
weights=weights.data,
returned=True)
assert np.isclose(avg.data, expected_avg)
assert np.isclose(scl.data, expected_scl)
scl.backward()
assert avg.grad is None
assert weights.grad is not None
def policyForward(self, data):
data = mg.Tensor(data)
x = self.conv1(data)
x = self.conv2(x)
x = relu(self.dense1(x.reshape(x.shape[0], -1)))
x = relu(self.dense2(x))
return self.dense3(x)
def test_where_condition_only_fwd(condition):
"""mygrad.where should merely mirror numpy.where when only `where(condition)`
is specified."""
tensor_condition = (mg.Tensor(condition) if isinstance(
condition, np.ndarray) else condition)
assert all(
np.all(x == y)
for x, y in zip(where(tensor_condition), np.where(condition)))
def targets(draw, scores):
as_tensor = draw(st.booleans())
out = draw(
st.lists(
st.integers(0, scores.shape[1] - 1),
min_size=len(scores),
max_size=len(scores),
))
return mg.Tensor(out) if as_tensor else np.array(out)
def __init__(self, dim_input, dim_recurrent, dim_output):
""" Initializes all layers needed for RNN
Parameters
----------
dim_input: int
Dimensionality of data passed to RNN (C)
dim_recurrent: int
Dimensionality of hidden state in RNN (D)
dim_output: int
Dimensionality of output of RNN (K)
"""
self.fc_x2h = dense(dim_input,
dim_recurrent,
weight_initializer=glorot_normal)
self.fc_h2h = dense(dim_recurrent,
dim_recurrent,
weight_initializer=glorot_normal,
bias=False)
self.fc_h2y = dense(dim_recurrent,
dim_output,
weight_initializer=glorot_normal)
self.Uz = mg.Tensor(
np.random.randn(dim_input * dim_recurrent).reshape(
dim_input, dim_recurrent))
self.Wz = mg.Tensor(
np.random.randn(dim_recurrent * dim_recurrent).reshape(
dim_recurrent, dim_recurrent))
self.bz = mg.Tensor(np.random.randn(dim_recurrent))
self.Ur = mg.Tensor(
np.random.randn(dim_input * dim_recurrent).reshape(
dim_input, dim_recurrent))
self.Wr = mg.Tensor(
np.random.randn(dim_recurrent * dim_recurrent).reshape(
dim_recurrent, dim_recurrent))
self.br = mg.Tensor(np.random.randn(dim_recurrent))
self.Uh = mg.Tensor(
np.random.randn(dim_input * dim_recurrent).reshape(
dim_input, dim_recurrent))
self.Wh = mg.Tensor(
np.random.randn(dim_recurrent * dim_recurrent).reshape(
dim_recurrent, dim_recurrent))
self.bh = mg.Tensor(np.random.randn(dim_recurrent))
def test_input_validation(args, data: st.DataObject):
kwargs = dict(x1=np.ones((3, 4)), y=mg.Tensor(1), margin=0.5)
if "x2" not in args:
args["x2"] = np.ones_like(kwargs["x1"])
kwargs.update(
(k, (data.draw(v, label=k)) if isinstance(v, st.SearchStrategy) else v)
for k, v in args.items())
with pytest.raises((ValueError, TypeError)):
margin_ranking_loss(**kwargs)
def policyForward(self, data):
data = mg.Tensor(data)
conv1 = conv(1,
20,
5,
5,
stride=1,
padding=0,
weight_initializer=glorot_uniform)
for i in range(len(self.weights) - 1):
data = mg.matmul(data, self.weights[i]) # hidden layers
data[data < 0] = 0 # ReLU
return mg.matmul(data, self.weights[-1]) # outNeurons
def test_op_tracks_graph():
"""Ensures that ``Operation.graph`` tracks operations as expected"""
x = mg.Tensor(1)
y = mg.Tensor(2)
z = x * y
assert z.creator.graph == {z.creator}
f = z + 2
assert f.creator.graph == {z.creator, f.creator}
h = z - f
assert h.creator.graph == {h.creator} | f.creator.graph
i = ((h + 3)**2) / 5
assert h.creator.graph < i.creator.graph
assert (len(i.creator.graph - h.creator.graph) == 3
), "should be {{Add, Subtract, Divide}}, but got {}".format(
i.creator.graph - h.creator.graph)
assert all(
isinstance(x, (Add, Power, Divide))
for x in i.creator.graph - h.creator.graph)
import pytest
from hypothesis import given
import mygrad as mg
from tests.wrappers.uber import backprop_test_factory, fwdprop_test_factory
def matmul_wrapper(*args, constant=False):
return mg.multi_matmul(args, constant)
def multi_matmul_slow(*arrays, **kwargs):
return functools.reduce(np.matmul, arrays)
@given(st.lists(st.just(mg.Tensor([0.0, 1.0])), min_size=0, max_size=1))
def test_input_validation_too_few_tensors(tensors: List[mg.Tensor]):
"""multi_matmul requires at least two input-tensors"""
with pytest.raises(ValueError):
mg.multi_matmul(tensors)
@given(
st.lists(hnp.array_shapes(min_dims=1), min_size=2).filter(
lambda shapes: any(not (1 <= len(x) <= 2) for x in shapes)))
def test_input_validation_large_dimensionality(shapes: List[Tuple[int, ...]]):
"""multi_matmul only operates on 1D and 2D tensors"""
tensors = [mg.ones(shape=shape) for shape in shapes]
with pytest.raises(ValueError):
mg.multi_matmul(tensors)
def __init__(self, in_dim, out_dim, h, c):
self.weight_ih = mg.Tensor(np.random.rand(4 * out_dim, in_dim))
self.weight_hh = mg.Tensor(np.random.rand(4 * out_dim, in_dim))
self.bias = mg.Tensor(np.zeros(4 * out_dim))
def __init__(self, in_dim, eps=1e-05):
self.gamma = mg.Tensor(np.random.rand(1, in_dim))
self.beta = mg.Tensor(np.random.rand(1, in_dim))
self.eps = eps
def __init__(self, n_inputs, n_neurons):
self.weight_ih = mg.Tensor(np.random.rand(n_inputs, n_neurons))
self.weight_hh = mg.Tensor(np.random.rand(n_neurons, n_neurons))
self.bias = mg.zeros((1, n_neurons))
def test_var_no_axis_bkwrd(x):
x = mg.Tensor(x, constant=False)
mg.var(x, axis=()).backward()
assert np.all(x.grad == np.zeros_like(x.data))
def test_var_no_axis_fwd(x):
x = mg.Tensor(x, constant=False)
o = mg.var(x, axis=())
assert np.all(o.data == np.zeros_like(x.data))
def test_multi_matmul(num_arrays, left_1d, right_1d, output_is_constant, data):
"""
Ensures that ``multi_matmul`` behaves identically to:
functools.reduce(mg.matmul, arrays)
Includes edge cases in which the 1st and last tensors in the sequence are 1D
"""
shape_endpoints = data.draw(
st.tuples(*[st.integers(1, 10) for i in range(num_arrays + 1)]),
label="endpoints",
)
shapes = [shape_endpoints[i:i + 2] for i in range(num_arrays)]
if left_1d:
shapes[0] = shapes[0][:0:-1]
if right_1d:
shapes[-1] = shapes[-1][:1]
constants = data.draw(
st.tuples(*[st.booleans() for i in range(num_arrays)]),
label="constants")
output_is_constant = output_is_constant or all(constants)
arrs = [
data.draw(
hnp.arrays(dtype=float,
shape=shapes[i],
elements=st.floats(0, 1e6)),
label="arr-{}".format(i),
) for i in range(num_arrays)
]
note("tensor shapes: {}".format([i.shape for i in arrs]))
arrs1 = [mg.Tensor(x, constant=const) for x, const in zip(arrs, constants)]
arrs2 = [x.__copy__() for x in arrs1]
actual = mg.multi_matmul(arrs1, constant=output_is_constant)
desired = multi_matmul_slow(arrs2)
assert_allclose(
actual.data,
desired.data,
atol=1e-6,
rtol=1e-6,
err_msg="`multi_matmul` does not produce the same result as "
"`functools.reduce(mg.matmul, arrays)`",
)
assert (actual.constant is output_is_constant
), "`multi_matmul` does not carry constant info properly"
if output_is_constant:
return
grad = data.draw(
hnp.arrays(shape=desired.shape,
dtype=float,
elements=st.floats(0, 1e6)))
(desired * grad).sum().backward()
(actual * grad).sum().backward()
for n, (const, arr1, arr2) in enumerate(zip(constants, arrs1, arrs2)):
assert (const is arr1.constant is arr2.constant
), "tensor-{}-constant was not set properly".format(n)
if const:
assert (
arr2.grad is None
), "tensor-{} is a constant, but its gradient is not `None`".format(
n)
else:
assert_allclose(
arr1.grad,
arr2.grad,
atol=1e-6,
rtol=1e-6,
err_msg="The gradients for tensor-{} for not match".format(n),
)
actual.null_gradients()
for n, arr1 in enumerate(arrs1):
assert arr1.grad is None, "tensor-{} did not get its gradient nulled".format(
n)
def __init__(self, layers):
self.weights = []
for i in range(len(layers) - 1):
self.weights.append(
mg.Tensor(np.random.randn(layers[i], layers[i + 1])))
def test_std_no_axis_fwd(x):
import mygrad as mg
x = mg.Tensor(x, constant=False)
o = mg.std(x, axis=())
assert np.all(o.data == np.zeros_like(x.data))
def test_std_no_axis_bkwrd(x):
import mygrad as mg
x = mg.Tensor(x, constant=False)
mg.std(x, axis=()).backward()
assert np.all(x.grad == np.zeros_like(x.data))
def main():
path = r"glove.6B.50d.txt.w2v"
glove = KeyedVectors.load_word2vec_format(path, binary=False)
# loads the json file
path_to_json = "captions_train2014.json"
with open(path_to_json, "rb") as f:
json_data = json.load(f)
resnet = unpickle.unpickle()
with open("idfs1.pkl", mode="rb") as idf:
idfs = pickle.load(idf)
with open("img_to_caption1.pkl", mode="rb") as cap:
img_to_caption = pickle.load(cap)
#with open("img_to_coco1.pkl", mode="rb") as coco:
#img_to_coco=pickle.load(coco)
model = Model()
model.dense1.weight = mg.Tensor(np.load('weight.npy'))
model.dense1.bias = mg.Tensor(np.load('bias.npy'))
optim = Adam(model.parameters)
batch_size = 100
for epoch_cnt in range(100):
idxs = list(resnet.keys())
np.random.shuffle(idxs)
for batch_cnt in range(0, len(idxs) // batch_size - 1):
batch_indices = idxs[(batch_cnt * batch_size):((batch_cnt + 1) *
batch_size)]
batch_indices2 = idxs[((batch_cnt + 1) *
batch_size):((batch_cnt + 2) * batch_size)]
# id1 = np.random.choice(list(resnet.keys()))
# print(id1)
id1 = batch_indices
# while id1 == id2:
id2 = batch_indices2
# print(type(resnet[id1]),type(img_to_caption[id1][0]),type(resnet[id2]))
good_image = resnet[id1[0]]
bad_image = resnet[id2[0]]
text = embed_text.se_text(img_to_caption[id1[0]][0], glove, idfs)
for i in id1[1:]:
good_image = np.vstack((good_image, resnet[i]))
text = np.vstack(
(text, embed_text.se_text(img_to_caption[i][0], glove,
idfs)))
for i in id2[1:]:
bad_image = np.vstack((bad_image, resnet[i]))
sim_to_good = cos_sim.cos_sim(model(good_image), text)
sim_to_bad = cos_sim.cos_sim(model(bad_image), text)
# compute the loss associated with our predictions(use softmax_cross_entropy)
loss = margin_ranking_loss(sim_to_good, sim_to_bad, 1, 0.1)
# back-propagate through your computational graph through your loss
loss.backward()
# compute the accuracy between the prediction and the truth
acc = accuracy(sim_to_good.data, sim_to_bad.data)
# execute gradient descent by calling step() of optim
optim.step()
# null your gradients
loss.null_gradients()
np.save('weight', model.dense1.parameters[0].data)
np.save('bias', model.dense1.parameters[1].data)
