def test_ranked_margin(shape, margin, data): x1 = data.draw( hnp.arrays(shape=shape, dtype=float, elements=st.floats(-1000, 1000)), label="x1", ) x2 = data.draw( hnp.arrays(shape=shape, dtype=float, elements=st.floats(-1000, 1000)), label="x2", ) y = data.draw( st.sampled_from((-1, 1)) | hnp.arrays( shape=shape[:1], dtype=hnp.integer_dtypes(), elements=st.sampled_from((-1, 1)), ), label="y", ) x1_copy = np.copy(x1) x2_copy = np.copy(x2) y_copy = np.copy(y) x1_dum = mg.Tensor(x1) x2_dum = mg.Tensor(x2) x1_real = mg.Tensor(x1) x2_real = mg.Tensor(x2) loss_dum = simple_loss(x1_dum, x2_dum, y, margin) loss_real = margin_ranking_loss(x1_real, x2_real, y, margin) assert_allclose(actual=loss_real.data, desired=loss_dum.data, err_msg="losses don't match") assert_array_equal(x1, x1_copy, err_msg="`x1` was mutated by forward") assert_array_equal(x2, x2_copy, err_msg="`x2` was mutated by forward") if isinstance(y, np.ndarray): assert_array_equal(y, y_copy, err_msg="`y` was mutated by forward") loss_dum.backward() loss_real.backward() assert_allclose(actual=x1_real.grad, desired=x1_dum.grad, err_msg="x1.grad doesn't match") assert_allclose(actual=x2_real.grad, desired=x2_dum.grad, err_msg="x2.grad doesn't match") assert_array_equal(x1, x1_copy, err_msg="`x1` was mutated by backward") assert_array_equal(x2, x2_copy, err_msg="`x2` was mutated by backward") if isinstance(y, np.ndarray): assert_array_equal(y, y_copy, err_msg="`y` was mutated by backward") loss_real.null_gradients() assert x1_real.grad is None assert x2_real.grad is None
def __init__(self, in_dim, out_dim, initializer=Uniform, bias=True): self.in_dim = in_dim self.out_dim = out_dim self.weights = mg.Tensor(np.random.rand(out_dim, in_dim)) if bias: self.bias = mg.Tensor(np.random.rand(out_dim)) else: self.bias = None
def test_average_scl(): x = mg.Tensor(np.array([1, 2, 3, 4])) weights = mg.Tensor(np.array([1, 2, 3, 4])) avg, scl = average(x, weights=weights, returned=True) expected_avg, expected_scl = np.average(x.data, weights=weights.data, returned=True) assert np.isclose(avg.data, expected_avg) assert np.isclose(scl.data, expected_scl) scl.backward() assert avg.grad is None assert weights.grad is not None
def policyForward(self, data): data = mg.Tensor(data) x = self.conv1(data) x = self.conv2(x) x = relu(self.dense1(x.reshape(x.shape[0], -1))) x = relu(self.dense2(x)) return self.dense3(x)
def test_where_condition_only_fwd(condition): """mygrad.where should merely mirror numpy.where when only `where(condition)` is specified.""" tensor_condition = (mg.Tensor(condition) if isinstance( condition, np.ndarray) else condition) assert all( np.all(x == y) for x, y in zip(where(tensor_condition), np.where(condition)))
def targets(draw, scores): as_tensor = draw(st.booleans()) out = draw( st.lists( st.integers(0, scores.shape[1] - 1), min_size=len(scores), max_size=len(scores), )) return mg.Tensor(out) if as_tensor else np.array(out)
def __init__(self, dim_input, dim_recurrent, dim_output): """ Initializes all layers needed for RNN Parameters ---------- dim_input: int Dimensionality of data passed to RNN (C) dim_recurrent: int Dimensionality of hidden state in RNN (D) dim_output: int Dimensionality of output of RNN (K) """ self.fc_x2h = dense(dim_input, dim_recurrent, weight_initializer=glorot_normal) self.fc_h2h = dense(dim_recurrent, dim_recurrent, weight_initializer=glorot_normal, bias=False) self.fc_h2y = dense(dim_recurrent, dim_output, weight_initializer=glorot_normal) self.Uz = mg.Tensor( np.random.randn(dim_input * dim_recurrent).reshape( dim_input, dim_recurrent)) self.Wz = mg.Tensor( np.random.randn(dim_recurrent * dim_recurrent).reshape( dim_recurrent, dim_recurrent)) self.bz = mg.Tensor(np.random.randn(dim_recurrent)) self.Ur = mg.Tensor( np.random.randn(dim_input * dim_recurrent).reshape( dim_input, dim_recurrent)) self.Wr = mg.Tensor( np.random.randn(dim_recurrent * dim_recurrent).reshape( dim_recurrent, dim_recurrent)) self.br = mg.Tensor(np.random.randn(dim_recurrent)) self.Uh = mg.Tensor( np.random.randn(dim_input * dim_recurrent).reshape( dim_input, dim_recurrent)) self.Wh = mg.Tensor( np.random.randn(dim_recurrent * dim_recurrent).reshape( dim_recurrent, dim_recurrent)) self.bh = mg.Tensor(np.random.randn(dim_recurrent))
def test_input_validation(args, data: st.DataObject): kwargs = dict(x1=np.ones((3, 4)), y=mg.Tensor(1), margin=0.5) if "x2" not in args: args["x2"] = np.ones_like(kwargs["x1"]) kwargs.update( (k, (data.draw(v, label=k)) if isinstance(v, st.SearchStrategy) else v) for k, v in args.items()) with pytest.raises((ValueError, TypeError)): margin_ranking_loss(**kwargs)
def policyForward(self, data): data = mg.Tensor(data) conv1 = conv(1, 20, 5, 5, stride=1, padding=0, weight_initializer=glorot_uniform) for i in range(len(self.weights) - 1): data = mg.matmul(data, self.weights[i]) # hidden layers data[data < 0] = 0 # ReLU return mg.matmul(data, self.weights[-1]) # outNeurons
def test_op_tracks_graph(): """Ensures that ``Operation.graph`` tracks operations as expected""" x = mg.Tensor(1) y = mg.Tensor(2) z = x * y assert z.creator.graph == {z.creator} f = z + 2 assert f.creator.graph == {z.creator, f.creator} h = z - f assert h.creator.graph == {h.creator} | f.creator.graph i = ((h + 3)**2) / 5 assert h.creator.graph < i.creator.graph assert (len(i.creator.graph - h.creator.graph) == 3 ), "should be {{Add, Subtract, Divide}}, but got {}".format( i.creator.graph - h.creator.graph) assert all( isinstance(x, (Add, Power, Divide)) for x in i.creator.graph - h.creator.graph)
import pytest from hypothesis import given import mygrad as mg from tests.wrappers.uber import backprop_test_factory, fwdprop_test_factory def matmul_wrapper(*args, constant=False): return mg.multi_matmul(args, constant) def multi_matmul_slow(*arrays, **kwargs): return functools.reduce(np.matmul, arrays) @given(st.lists(st.just(mg.Tensor([0.0, 1.0])), min_size=0, max_size=1)) def test_input_validation_too_few_tensors(tensors: List[mg.Tensor]): """multi_matmul requires at least two input-tensors""" with pytest.raises(ValueError): mg.multi_matmul(tensors) @given( st.lists(hnp.array_shapes(min_dims=1), min_size=2).filter( lambda shapes: any(not (1 <= len(x) <= 2) for x in shapes))) def test_input_validation_large_dimensionality(shapes: List[Tuple[int, ...]]): """multi_matmul only operates on 1D and 2D tensors""" tensors = [mg.ones(shape=shape) for shape in shapes] with pytest.raises(ValueError): mg.multi_matmul(tensors)
def __init__(self, in_dim, out_dim, h, c): self.weight_ih = mg.Tensor(np.random.rand(4 * out_dim, in_dim)) self.weight_hh = mg.Tensor(np.random.rand(4 * out_dim, in_dim)) self.bias = mg.Tensor(np.zeros(4 * out_dim))
def __init__(self, in_dim, eps=1e-05): self.gamma = mg.Tensor(np.random.rand(1, in_dim)) self.beta = mg.Tensor(np.random.rand(1, in_dim)) self.eps = eps
def __init__(self, n_inputs, n_neurons): self.weight_ih = mg.Tensor(np.random.rand(n_inputs, n_neurons)) self.weight_hh = mg.Tensor(np.random.rand(n_neurons, n_neurons)) self.bias = mg.zeros((1, n_neurons))
def test_var_no_axis_bkwrd(x): x = mg.Tensor(x, constant=False) mg.var(x, axis=()).backward() assert np.all(x.grad == np.zeros_like(x.data))
def test_var_no_axis_fwd(x): x = mg.Tensor(x, constant=False) o = mg.var(x, axis=()) assert np.all(o.data == np.zeros_like(x.data))
def test_multi_matmul(num_arrays, left_1d, right_1d, output_is_constant, data): """ Ensures that ``multi_matmul`` behaves identically to: functools.reduce(mg.matmul, arrays) Includes edge cases in which the 1st and last tensors in the sequence are 1D """ shape_endpoints = data.draw( st.tuples(*[st.integers(1, 10) for i in range(num_arrays + 1)]), label="endpoints", ) shapes = [shape_endpoints[i:i + 2] for i in range(num_arrays)] if left_1d: shapes[0] = shapes[0][:0:-1] if right_1d: shapes[-1] = shapes[-1][:1] constants = data.draw( st.tuples(*[st.booleans() for i in range(num_arrays)]), label="constants") output_is_constant = output_is_constant or all(constants) arrs = [ data.draw( hnp.arrays(dtype=float, shape=shapes[i], elements=st.floats(0, 1e6)), label="arr-{}".format(i), ) for i in range(num_arrays) ] note("tensor shapes: {}".format([i.shape for i in arrs])) arrs1 = [mg.Tensor(x, constant=const) for x, const in zip(arrs, constants)] arrs2 = [x.__copy__() for x in arrs1] actual = mg.multi_matmul(arrs1, constant=output_is_constant) desired = multi_matmul_slow(arrs2) assert_allclose( actual.data, desired.data, atol=1e-6, rtol=1e-6, err_msg="`multi_matmul` does not produce the same result as " "`functools.reduce(mg.matmul, arrays)`", ) assert (actual.constant is output_is_constant ), "`multi_matmul` does not carry constant info properly" if output_is_constant: return grad = data.draw( hnp.arrays(shape=desired.shape, dtype=float, elements=st.floats(0, 1e6))) (desired * grad).sum().backward() (actual * grad).sum().backward() for n, (const, arr1, arr2) in enumerate(zip(constants, arrs1, arrs2)): assert (const is arr1.constant is arr2.constant ), "tensor-{}-constant was not set properly".format(n) if const: assert ( arr2.grad is None ), "tensor-{} is a constant, but its gradient is not `None`".format( n) else: assert_allclose( arr1.grad, arr2.grad, atol=1e-6, rtol=1e-6, err_msg="The gradients for tensor-{} for not match".format(n), ) actual.null_gradients() for n, arr1 in enumerate(arrs1): assert arr1.grad is None, "tensor-{} did not get its gradient nulled".format( n)
def __init__(self, layers): self.weights = [] for i in range(len(layers) - 1): self.weights.append( mg.Tensor(np.random.randn(layers[i], layers[i + 1])))
def test_std_no_axis_fwd(x): import mygrad as mg x = mg.Tensor(x, constant=False) o = mg.std(x, axis=()) assert np.all(o.data == np.zeros_like(x.data))
def test_std_no_axis_bkwrd(x): import mygrad as mg x = mg.Tensor(x, constant=False) mg.std(x, axis=()).backward() assert np.all(x.grad == np.zeros_like(x.data))
def main(): path = r"glove.6B.50d.txt.w2v" glove = KeyedVectors.load_word2vec_format(path, binary=False) # loads the json file path_to_json = "captions_train2014.json" with open(path_to_json, "rb") as f: json_data = json.load(f) resnet = unpickle.unpickle() with open("idfs1.pkl", mode="rb") as idf: idfs = pickle.load(idf) with open("img_to_caption1.pkl", mode="rb") as cap: img_to_caption = pickle.load(cap) #with open("img_to_coco1.pkl", mode="rb") as coco: #img_to_coco=pickle.load(coco) model = Model() model.dense1.weight = mg.Tensor(np.load('weight.npy')) model.dense1.bias = mg.Tensor(np.load('bias.npy')) optim = Adam(model.parameters) batch_size = 100 for epoch_cnt in range(100): idxs = list(resnet.keys()) np.random.shuffle(idxs) for batch_cnt in range(0, len(idxs) // batch_size - 1): batch_indices = idxs[(batch_cnt * batch_size):((batch_cnt + 1) * batch_size)] batch_indices2 = idxs[((batch_cnt + 1) * batch_size):((batch_cnt + 2) * batch_size)] # id1 = np.random.choice(list(resnet.keys())) # print(id1) id1 = batch_indices # while id1 == id2: id2 = batch_indices2 # print(type(resnet[id1]),type(img_to_caption[id1][0]),type(resnet[id2])) good_image = resnet[id1[0]] bad_image = resnet[id2[0]] text = embed_text.se_text(img_to_caption[id1[0]][0], glove, idfs) for i in id1[1:]: good_image = np.vstack((good_image, resnet[i])) text = np.vstack( (text, embed_text.se_text(img_to_caption[i][0], glove, idfs))) for i in id2[1:]: bad_image = np.vstack((bad_image, resnet[i])) sim_to_good = cos_sim.cos_sim(model(good_image), text) sim_to_bad = cos_sim.cos_sim(model(bad_image), text) # compute the loss associated with our predictions(use softmax_cross_entropy) loss = margin_ranking_loss(sim_to_good, sim_to_bad, 1, 0.1) # back-propagate through your computational graph through your loss loss.backward() # compute the accuracy between the prediction and the truth acc = accuracy(sim_to_good.data, sim_to_bad.data) # execute gradient descent by calling step() of optim optim.step() # null your gradients loss.null_gradients() np.save('weight', model.dense1.parameters[0].data) np.save('bias', model.dense1.parameters[1].data)