def search_phrase(query, annotations, image_id2id, glove, vectors_512, inds_img, weight, bias, loaded_file): query = query.lower() punc_regex = re.compile('[{}]'.format(re.escape(string.punctuation))) def strip_punc(corpus): return punc_regex.sub('', corpus) query = strip_punc(query) query_components = query.split() N = len(annotations) queries_in_dict = [] for word in query_components: if word in glove: nt = 1 for caption in annotations: nt += word in set(caption.split()) idf = np.log10(N / nt) queries_in_dict.append(glove[word] * idf) print("Got the idf") final_embedding = np.mean(np.vstack(queries_in_dict), axis=0) # Find the cosine distance between the 50D vectors and the embeddings #s = np.sum(vectors_50, axis = 1) print(type(vectors_512)) print(type(vectors_512[0])) print(type(weight)) vectors_50 = mg.matmul(vectors_512, weight) + bias #print(vectors_50[:k]) #print(vectors_50[21697]) print("shape1", vectors_50.shape) x = np.arange(10).reshape(5, 2) print(np.sum(x, axis=1)) a = vectors_50**2 sum_s = a.sum(axis=1) vectors_50 = vectors_50 / mg.sqrt(sum_s).reshape(vectors_50.shape[0], 1) print("Got sum 1", vectors_50.shape) #s = np.sum(final_embedding) final_embedding = final_embedding / np.sqrt(np.sum(final_embedding**2)) print("Printing images") cos = np.dot(vectors_50.data, final_embedding) k = 4 max_vals = np.argsort(cos)[-k:] fig, ax = plt.subplots(2, 2) for ind, ima in enumerate(inds_img[max_vals]): url = loaded_file["images"][image_id2id[ima]]['coco_url'] img = get_image(url) ax[ind // 2, ind % 2].imshow(img)
def __call__(self, x): """ Perform the forward-pass of the densely-connected layer over `x`. Parameters ---------- x : Union[numpy.ndarray, mygrad.Tensor], shape=(N, D) The input to pass through the layer. Returns ------- mygrad.Tensor The result of applying the dense layer wx + b. """ out = matmul(x, self.weight) return out + self.bias if self.bias is not None else out
def cosine_dist(d1, d2): """ takes 2 vectors shape (50,) and returns cosine distance Parameters ---------- d1 : numpy array first d vector d2 : numpy array 2nd d vector Returns ------- float cosine distance of d1 and d2 """ return mg.matmul(d1, d2)
def test_gru_fwd(X, D, dropout, data: st.DataObject): T, N, C = X.shape Wz, Wr, Wh = data.draw( hnp.arrays(shape=(3, D, D), dtype=float, elements=st.floats(-10.0, 10.0)), label="Wz, Wr, Wh", ) Uz, Ur, Uh = data.draw( hnp.arrays(shape=(3, C, D), dtype=float, elements=st.floats(-10.0, 10.0)), label="Uz, Ur, Uh", ) bz, br, bh = data.draw( hnp.arrays(shape=(3, D), dtype=float, elements=st.floats(-10.0, 10.0)), label="bz, br, bh", ) V = data.draw( hnp.arrays(shape=(D, C), dtype=float, elements=st.floats(-10.0, 10.0)), label="V", ) s0 = np.zeros((N, D), dtype=float) X = Tensor(X) X2 = X.__copy__() Wz = Tensor(Wz) Wz2 = Wz.__copy__() Uz = Tensor(Uz) Uz2 = Uz.__copy__() bz = Tensor(bz) bz2 = bz.__copy__() Wr = Tensor(Wr) Wr2 = Wr.__copy__() Ur = Tensor(Ur) Ur2 = Ur.__copy__() br = Tensor(br) br2 = br.__copy__() Wh = Tensor(Wh) Wh2 = Wh.__copy__() Uh = Tensor(Uh) Uh2 = Uh.__copy__() bh = Tensor(bh) bh2 = bh.__copy__() V = Tensor(V) V2 = V.__copy__() s0 = Tensor(s0) s2 = s0.__copy__() s = gru(X, Uz, Wz, bz, Ur, Wr, br, Uh, Wh, bh, dropout=dropout, constant=True) o = matmul(s[1:], V) ls = o.sum() assert s.constant is True if dropout: for d in [ s.creator._dropUr, s.creator._dropUz, s.creator._dropUh, s.creator._dropWr, s.creator._dropWz, s.creator._dropWh, ]: assert np.all(np.logical_or(d == 1 / (1 - dropout), d == 0)) stt = s2 all_s = [s0.data] ls2 = 0 if dropout: Wz2d = s.creator._dropWz * Wz2 Wr2d = s.creator._dropWr * Wr2 Wh2d = s.creator._dropWh * Wh2 else: Wz2d = Wz2 Wr2d = Wr2 Wh2d = Wh2 for n, x in enumerate(X2): if not dropout: z = sigmoid(matmul(x, Uz2) + matmul(stt, Wz2d) + bz2) r = sigmoid(matmul(x, Ur2) + matmul(stt, Wr2d) + br2) h = tanh(matmul(x, Uh2) + matmul((r * stt), Wh2d) + bh2) else: z = sigmoid((s.creator._dropUz[0] * matmul(x, Uz2)) + matmul(stt, Wz2d) + bz2) r = sigmoid((s.creator._dropUr[0] * matmul(x, Ur2)) + matmul(stt, Wr2d) + br2) h = tanh((s.creator._dropUh[0] * matmul(x, Uh2)) + matmul((r * stt), Wh2d) + bh2) stt = (1 - z) * h + z * stt all_s.append(stt) o = matmul(stt, V2) ls2 += o.sum() tolerances = dict(atol=1e-5, rtol=1e-5) rec_s_dat = np.stack([i.data for i in all_s]) assert_allclose(ls.data, ls2.data, **tolerances) assert_allclose(rec_s_dat, s.data, **tolerances) assert_allclose(Wz.data, Wz2.data, **tolerances) assert_allclose(Wr.data, Wr2.data, **tolerances) assert_allclose(Wh.data, Wh2.data, **tolerances) assert_allclose(Uz.data, Uz2.data, **tolerances) assert_allclose(Ur.data, Ur2.data, **tolerances) assert_allclose(Uh.data, Uh2.data, **tolerances) assert_allclose(bz.data, bz2.data, **tolerances) assert_allclose(br.data, br2.data, **tolerances) assert_allclose(bh.data, bh2.data, **tolerances) assert_allclose(V.data, V2.data, **tolerances) assert_allclose(X.data, X2.data, **tolerances) ls.null_gradients() for x in [s, Wz, Wr, Wh, bz, br, bh, X, Uz, Ur, Uh, V]: assert x.grad is None
def test_gru_backward( data: st.DataObject, X: np.ndarray, D: int, bp_lim: bool, dropout: bool, U_constants: Tuple[bool, bool, bool], W_constants: Tuple[bool, bool, bool], b_constants: Tuple[bool, bool, bool], X_constant: bool, V_constant: bool, ): tolerances = dict(atol=1e-5, rtol=1e-5) T, N, C = X.shape Wz, Wr, Wh = data.draw( hnp.arrays(shape=(3, D, D), dtype=float, elements=st.floats(-10.0, 10.0)), label="Wz, Wr, Wh", ) Uz, Ur, Uh = data.draw( hnp.arrays(shape=(3, C, D), dtype=float, elements=st.floats(-10.0, 10.0)), label="Uz, Ur, Uh", ) bz, br, bh = data.draw( hnp.arrays(shape=(3, D), dtype=float, elements=st.floats(-10.0, 10.0)), label="bz, br, bh", ) V = data.draw( hnp.arrays(shape=(D, C), dtype=float, elements=st.floats(-10.0, 10.0)), label="V", ) s0 = np.zeros((N, D), dtype=float) X = Tensor(X, constant=X_constant) X2 = X.__copy__() Wz = Tensor(Wz, constant=W_constants[0]) Wz2 = Wz.__copy__() Uz = Tensor(Uz, constant=U_constants[0]) Uz2 = Uz.__copy__() bz = Tensor(bz, constant=b_constants[0]) bz2 = bz.__copy__() Wr = Tensor(Wr, constant=W_constants[1]) Wr2 = Wr.__copy__() Ur = Tensor(Ur, constant=U_constants[1]) Ur2 = Ur.__copy__() br = Tensor(br, constant=b_constants[1]) br2 = br.__copy__() Wh = Tensor(Wh, constant=W_constants[2]) Wh2 = Wh.__copy__() Uh = Tensor(Uh, constant=U_constants[2]) Uh2 = Uh.__copy__() bh = Tensor(bh, constant=b_constants[2]) bh2 = bh.__copy__() V = Tensor(V, constant=V_constant) V2 = V.__copy__() s0 = Tensor(s0) s2 = s0.__copy__() # bp_lim = len(X) - 1 should behave the same as no bp-lim s = gru( X, Uz, Wz, bz, Ur, Wr, br, Uh, Wh, bh, dropout=dropout, constant=False, bp_lim=len(X) - 1 if bp_lim else None, ) o = matmul(s[1:], V) ls = o.sum() ls.backward() stt = s2 all_s = [s0.data] ls2 = 0 if dropout: Wz2d = s.creator._dropWz * Wz2 Wr2d = s.creator._dropWr * Wr2 Wh2d = s.creator._dropWh * Wh2 else: Wz2d = Wz2 Wr2d = Wr2 Wh2d = Wh2 for n, x in enumerate(X2): if not dropout: z = sigmoid(matmul(x, Uz2) + matmul(stt, Wz2d) + bz2) r = sigmoid(matmul(x, Ur2) + matmul(stt, Wr2d) + br2) h = tanh(matmul(x, Uh2) + matmul((r * stt), Wh2d) + bh2) else: z = sigmoid((s.creator._dropUz[0] * matmul(x, Uz2)) + matmul(stt, Wz2d) + bz2) r = sigmoid((s.creator._dropUr[0] * matmul(x, Ur2)) + matmul(stt, Wr2d) + br2) h = tanh((s.creator._dropUh[0] * matmul(x, Uh2)) + matmul((r * stt), Wh2d) + bh2) stt = (1 - z) * h + z * stt all_s.append(stt) o = matmul(stt, V2) ls2 += o.sum() ls2.backward() rec_s_grad = np.stack([i.grad for i in all_s[1:]]) if not s.constant: assert_allclose(rec_s_grad, s.grad, **tolerances) else: assert s.grad is None if not Wz.constant: assert_allclose(Wz.grad, Wz2.grad, **tolerances) else: assert Wz.grad is None if not Wr.constant: assert_allclose(Wr.grad, Wr2.grad, **tolerances) else: assert Wr.grad is None if not Wh.constant: assert_allclose(Wh.grad, Wh2.grad, **tolerances) else: assert Wh.grad is None if not Uz.constant: assert_allclose(Uz.grad, Uz2.grad, **tolerances) else: assert Uz.grad is None if not Ur.constant: assert_allclose(Ur.grad, Ur2.grad, **tolerances) else: assert Ur.grad is None if not Uh.constant: assert_allclose(Uh.grad, Uh2.grad, **tolerances) else: assert Uh.grad is None if not bz.constant: assert_allclose(bz.grad, bz2.grad, **tolerances) else: assert bz.grad is None if not br.constant: assert_allclose(br.grad, br2.grad, **tolerances) else: assert br.grad is None if not bh.constant: assert_allclose(bh.grad, bh2.grad, **tolerances) else: assert bh.grad is None if not V.constant: assert_allclose(V.grad, V2.grad, **tolerances) else: assert V.grad is None if not X.constant: assert_allclose(X.grad, X2.grad, **tolerances) else: assert X.grad is None ls.null_gradients() ls2.null_gradients() for x in [s, Wz, Wr, Wh, bz, br, bh, X, Uz, Ur, Uh, V]: assert x.grad is None
def dense(x, y): return matmul(x, y)
def dense(x, y): return matmul(x, y) @given(st.data())
def test_recurrent(data): X = data.draw( hnp.arrays( shape=hnp.array_shapes(max_side=5, min_dims=3, max_dims=3), dtype=float, elements=st.floats(-10, 10), )) T, N, C = X.shape D = data.draw(st.sampled_from(list(range(1, 5)))) s0 = data.draw( hnp.arrays(shape=(N, D), dtype=float, elements=st.floats(0.0, 0.0))) W = data.draw( hnp.arrays(shape=(D, D), dtype=float, elements=st.floats(-10.0, 10.0))) U = data.draw( hnp.arrays(shape=(C, D), dtype=float, elements=st.floats(-10.0, 10.0))) V = data.draw( hnp.arrays(shape=(D, C), dtype=float, elements=st.floats(-10.0, 10.0))) X = Tensor(X) X2 = X.__copy__() W = Tensor(W) W2 = W.__copy__() U = Tensor(U) U2 = U.__copy__() V = Tensor(V) V2 = V.__copy__() s0 = Tensor(s0) s2 = s0.__copy__() s = simple_RNN(X, U, W) o = matmul(s[1:], V) ls = o.sum() ls.backward() stt = s2 all_s = [s0.data] ls2 = 0 for n, x in enumerate(X2): stt = tanh(matmul(x, U2) + matmul(stt, W2)) all_s.append(stt) o = matmul(stt, V2) ls2 += o.sum() ls2.backward() rec_s_dat = np.stack([i.data for i in all_s]) rec_s_grad = np.stack([i.grad for i in all_s[1:]]) assert_allclose(rec_s_dat, s.data, atol=1e-5, rtol=1e-5) assert_allclose(rec_s_grad, s.grad[1:], atol=1e-5, rtol=1e-5) assert_allclose(ls.data, ls2.data, atol=1e-5, rtol=1e-5) assert_allclose(W.data, W2.data, atol=1e-5, rtol=1e-5) assert_allclose(W.grad, W2.grad, atol=1e-5, rtol=1e-5) assert_allclose(U.data, U2.data, atol=1e-5, rtol=1e-5) assert_allclose(U.grad, U2.grad, atol=1e-5, rtol=1e-5) assert_allclose(V.data, V2.data, atol=1e-5, rtol=1e-5) assert_allclose(V.grad, V2.grad, atol=1e-5, rtol=1e-5) assert_allclose(X.data, X2.data, atol=1e-5, rtol=1e-5) assert_allclose(X.grad, X2.grad, atol=1e-5, rtol=1e-5) ls.null_gradients() for x in [s, W, X, U, V]: assert x.grad is None
def __call__(self, x, h): Y0 = mg.nnet.tanh(mg.matmul(x, self.weight_ih)) - self.bias Y1 = mg.nnet.tanh( mg.matmul(Y0, self.weight_hh) - mg.matmul(h, self.weight_ih) + self.bias) return Y0, Y1
def __call__(self, X): X = relu(mg.matmul(X, self.w1, True) + self.b1) X = relu(mg.matmul(X, self.w2, True) + self.b2) X = mg.matmul(X, self.w3, True) + self.b3 return mg.nnet.activations.softmax(X, constant=True)