Ejemplo n.º 1
0
 def __init__(self, base, override, maskid=-1, which=None, **kw):
     assert (base.outdim == override.outdim)
     baseindexes = Val(np.asarray(sorted(base.D.values()), dtype="int32"))
     basevar = base(baseindexes)
     if which is None:
         ad = {
             v: override.D[k] if k in override.D else 0
             for k, v in base.D.items()
         }
     else:
         ad = {
             base.D[k]: override.D[k] if k in override.D else 0
             for k in which
         }
     valval = np.zeros((max(ad.keys()) + 1, ), dtype="int32")
     for i in range(valval.shape[0]):
         valval[i] = ad[i] if i in ad else 0
     overrideindexes = Val(valval)
     overridevar = override(overrideindexes)
     overridemask = np.repeat(valval[:, None], base.outdim, axis=1)
     v = Switch(overridevar, basevar, overridemask)
     super(NewOverriddenWordEmb, self).__init__(worddic=base.D,
                                                value=v(),
                                                dim=base.outdim,
                                                maskid=maskid)
Ejemplo n.º 2
0
 def test_set_lr(self):
     attdist = LinearDistance(110, 110, 100)
     encdec = SimpleSeqEncDecAtt(inpvocsize=19,
                                 outvocsize=17,
                                 outconcat=False,
                                 encdim=110,
                                 decdim=110,
                                 attdist=attdist)
     encdec.dec.set_lr(0.1)
     encdec.dec.attention.set_lr(0.5)  # TODO
     encdata = np.random.randint(0, 19, (2, 5))
     decdata = np.random.randint(0, 17, (2, 5))
     o = encdec(Val(encdata), Val(decdata))
     #print "\n".join(["{}: {}".format(x, x.lrmul) for x in o.allparams])
     #print "\n".join(["{}: {}".format(x, x.lrmul) for x in o.allparams])
     encparams = encdec.enc.get_params()
     decparams = encdec.dec.get_params()
     attparams = encdec.dec.attention.get_params()
     print "\n".join(["{}: {}".format(x, x.lrmul)
                      for x in encparams]) + "\n"
     print "\n".join(["{}: {}".format(x, x.lrmul)
                      for x in decparams]) + "\n"
     for x in encparams:
         self.assertEqual(x.lrmul, 1.0)
     for x in decparams:
         if x not in attparams:
             self.assertEqual(x.lrmul, 0.1)
         else:
             self.assertEqual(x.lrmul, 0.5)
Ejemplo n.º 3
0
 def test_reverse(self):
     xval = np.random.randint(0, 5, (2, 3, 4))
     x = Val(xval)
     y = x.reverse(1)
     yval = xval[:, ::-1, :]
     self.assertTrue(np.allclose(yval, y.eval()))
     y = x.reverse(0, 1)
     yval = xval[::-1, ::-1, :]
     self.assertTrue(np.allclose(yval, y.eval()))
Ejemplo n.º 4
0
 def __init__(self,
              indim=1000,
              dim=50,
              value=None,
              normalize=False,
              trainfrac=1.0,
              **kw):
     super(VectorEmbed, self).__init__(indim, dim, **kw)
     self.dim = dim
     self.indim = indim
     self.trainfrac = trainfrac
     if value is None:
         self.W = param((indim, dim), lrmul=self.trainfrac,
                        name="embedder").glorotuniform()
     else:
         if trainfrac == 0.0:
             self.W = Val(value, name="embedder_val")
         else:
             self.W = Parameter(value,
                                lrmul=self.trainfrac,
                                name="embedder")
     if normalize:
         self.W = self.W.normalize(axis=1)
     # assertions
     assert (self.W.d.get_value().shape == (self.indim, self.dim))
Ejemplo n.º 5
0
 def test_max_pool_masked(self):
     xval = np.random.random((100, 20, 50)).astype("float32")
     maskid = np.random.randint(1, 18, (100, ))
     mask = np.ones((xval.shape[:2]))
     for i in range(mask.shape[0]):
         mask[i, maskid[i]:] = 0
     #xval[:, :, -1] = 100
     x = Val(xval)
     x.mask = Val(mask)
     pool = GlobalPool1D(mode="max")
     pred = pool(x)
     predval = pred.eval()
     xval = xval - 1e9 * np.tensordot(1 - mask, np.ones(
         (xval.shape[-1], )), 0)
     predvalexp = np.max(xval, axis=1)
     self.assertTrue(np.allclose(predval, predvalexp))
Ejemplo n.º 6
0
 def test_set_lr(self):
     lin = Linear(indim=10, dim=15)
     lin.set_lr(0.123)
     o = lin(Val(0))
     #print ["{}: {}".format(x, x.lrmul) for x in o.allparams]
     for x in o.allparams:
         self.assertEqual(x.lrmul, 0.123)
Ejemplo n.º 7
0
 def test_total_pool(self):
     xval = np.random.random((10, 5))
     x = Val(xval)
     xpooled = MaxPool((None, ), axis=(1, ))(x)
     self.assertTrue(np.allclose(xpooled.eval(), xval.max(axis=1)))
     xpooled = SumPool((None, ), axis=(1, ))(x)
     self.assertTrue(np.allclose(xpooled.eval(), xval.sum(axis=1)))
Ejemplo n.º 8
0
 def test_compound_var(self):
     aval = np.zeros((10, 10))
     bval = np.ones((10, 10))
     maskval = np.repeat(np.asarray([[0, 1, 0, 0, 1, 0, 0, 1, 1, 1]]).T,
                         10,
                         axis=1)
     print maskval
     print aval * maskval + bval * (1 - maskval)
     a = Val(aval)
     b = Val(bval)
     mask = Val(maskval)
     cv = Switch(a, b, mask)
     cvpred = cv().eval()
     print cvpred
     self.assertTrue(
         np.allclose(cvpred, aval * maskval + bval * (1 - maskval)))
Ejemplo n.º 9
0
 def setvalue(self, v):
     if isinstance(v, Var):
         self.W = v
     else:
         if self.trainfrac == 0.0:
             self.W = Val(v, name="embedder_val")
         else:
             self.W = Parameter(v, lrmul=self.trainfrac, name="embedder")
         self.indim, self.outdim = v.shape
Ejemplo n.º 10
0
 def test_output_mask_strided(self):
     xval = np.random.random((100, 20, 50)).astype("float32")
     maskid = np.random.randint(3, 20, (100, ))
     mask = np.ones((xval.shape[:2]))
     for i in range(mask.shape[0]):
         mask[i, maskid[i]:] = 0
     conv = Conv1D(indim=50,
                   outdim=40,
                   window=5,
                   stride=4,
                   border_mode="valid")
     x = Val(xval)
     x.mask = Val(mask)
     pred = conv(x)
     predmask = pred.mask
     print predmask.eval().shape
     print predmask.eval()[:5]
     print mask[:5]
Ejemplo n.º 11
0
 def test_high_d_pool(self):
     xval = np.random.random((10, 6, 18, 14))
     x = Val(xval)
     xpooled = AvgPool((2, 2), axis=(2, 1))(x)
     self.assertEqual(xpooled.eval().shape, (10, 3, 9, 14))
     xpooled = AvgPool((2, 3), axis=(1, 2))(x)
     self.assertEqual(xpooled.eval().shape, (10, 3, 6, 14))
     xpooled = AvgPool((2, 3), axis=(2, 1))(x)
     self.assertEqual(xpooled.eval().shape, (10, 2, 9, 14))
Ejemplo n.º 12
0
 def test_enc_mask(self):
     xval = np.random.randint(1, 200, (100, 20)).astype("int32")
     maskid = np.random.randint(0, 5, (100, ))
     for i in range(xval.shape[0]):
         xval[i, maskid[i]:] = 0
     x = Val(xval)
     enc = CNNSeqEncoder(indim=200, inpembdim=50, innerdim=5, maskid=0)
     pred = enc(x)
     #print pred.mask.eval().shape
     predval = pred.eval()
     print predval.shape
Ejemplo n.º 13
0
 def test_output_shape_masked(self):
     xval = np.random.random((100, 20, 50)).astype("float32")
     maskid = np.random.randint(3, 20, (100, ))
     mask = np.ones((xval.shape[:2]))
     for i in range(mask.shape[0]):
         mask[i, maskid[i]:] = 0
     conv = Conv1D(indim=50, outdim=40, window=5)
     x = Val(xval)
     x.mask = Val(mask)
     pred = conv(x)
     predmask = pred.mask
     predval = pred.eval()
     predvalmask = (predval != 0.0) * 1
     predvalexpmask = np.ones_like(predvalmask)
     for i in range(predvalexpmask.shape[0]):
         predvalexpmask[i,
                        min(maskid[i] + 2, predvalexpmask.shape[1]):, :] = 0
     self.assertTrue(np.sum(predvalexpmask - predvalmask) == 0)
     self.assertEqual(predval.shape[:2], xval.shape[:2])
     self.assertEqual(predval.shape[2], 40)
Ejemplo n.º 14
0
 def __init__(self, encdim, invocsize, outvocsize, innerdim, seqlen, **kw):
     super(idx2seqTheano, self).__init__(**kw)
     self.encdim = encdim
     self.invocsize = invocsize
     self.outvocsize = outvocsize
     self.innerdim = innerdim
     self.seqlen = seqlen
     self.wordemb = param((invocsize, encdim)).uniform()
     self.idxtovec = Val(np.eye(outvocsize, outvocsize))
     self.rnu_w = param((encdim + outvocsize, innerdim)).uniform()
     self.rnu_u = param((innerdim, innerdim)).uniform()
     self.outpf = theano.tensor.tanh
     self.olin = param((innerdim, outvocsize)).uniform()
     self.ownparams = [self.wordemb, self.rnu_u, self.rnu_w, self.olin]
Ejemplo n.º 15
0
 def __init__(self, block=None, data=None, indim=200, outdim=50, **kw):
     assert(block is not None)
     ourdata = []
     if not issequence(data):
         data = [data]
     for datae in data:
         if not isinstance(datae, (Var, Val)) and datae is not None:
             ourdata.append(Val(datae))
         else:
             ourdata.append(datae)
     assert(isinstance(block, Block))
     self.data = ourdata
     super(MemoryBlock, self).__init__(indim, outdim, **kw)      # outdim = outdim of the contained block
     self.payload = block
     self.innervar = self.payload(*self.data) if None not in data else None    # innervar: (indim, outdim)
Ejemplo n.º 16
0
    def get_init_info(
        self,
        inpseq,
        batsize,
        maskseq=None
    ):  # TODO: must evaluate enc here, in place, without any side effects
        """
        VERY DIFFERENT FROM THE PURELY SYMBOLIC GET_INIT_INFO IN REAL REC BLOCKS !!!
        This one is used in decoder/prediction
        """
        enco, allenco, encmask = self.enc.predict(inpseq, mask=maskseq)

        if self.statetrans is not None:
            topstate = self.statetrans.predict(
                enco, allenco
            )  # this gives unused input warning in theano - it's normal
            initstates = [topstate]
        else:
            initstates = batsize
        return self.dec.get_init_info(
            Val(allenco),
            [Val(x)
             for x in initstates] if issequence(initstates) else initstates,
            encmask=Val(encmask))
Ejemplo n.º 17
0
 def test_multilevel_set_lr(self):
     l1 = Linear(10, 11)
     l2 = Linear(11, 12)
     l3 = Linear(12, 13)
     s = stack(l1, l2, l3)
     s[1].set_lr(0.5)
     s[2].set_lr(0.1)
     o = s(Val(0))
     l1o = s[0](Val(0))
     l2o = s[1](Val(0))
     l3o = s[2](Val(0))
     print["{}: {}".format(x, x.lrmul) for x in o.allparams]
     for x in o.allparams:
         if x in l1o.allparams:
             self.assertEqual(x.lrmul, 1.0)
         elif x in l2o.allparams:
             self.assertEqual(x.lrmul, 0.5)
         elif x in l3o.allparams:
             self.assertEqual(x.lrmul, 0.1)
     s.set_lr(0.21)
     o = s(Val(0))
     print["{}: {}".format(x, x.lrmul) for x in o.allparams]
     for x in o.allparams:
         self.assertEqual(x.lrmul, 0.21)
Ejemplo n.º 18
0
 def test_bidir(self):
     m = RNNSeqEncoder(indim=20,
                       inpembdim=5,
                       innerdim=(10, 10),
                       bidir=True,
                       maskid=0).with_outputs()
     xval = np.random.randint(1, 20, (7, 3))
     xval = np.concatenate([xval, np.zeros_like(xval)], axis=1)
     x = Val(xval)
     fmp, mp = m(x)
     fmpval, mpval = fmp.eval(), mp.eval()
     self.assertTrue(np.allclose(fmpval[:, :10], mpval[:, -1, :10]))
     self.assertTrue(np.allclose(fmpval[:, 10:], mpval[:, 0, 10:]))
     mpm = mp.mask
     self.assertEqual(np.sum(mpm.eval() - xval > 0), 0)
Ejemplo n.º 19
0
 def __init__(self, base, augment, **kw):
     assert (base.outdim == augment.outdim)
     super(AugmentedWordEmb, self).__init__(worddic=base.D,
                                            value=False,
                                            dim=base.outdim,
                                            normalize=base.normalize,
                                            trainfrac=base.trainfrac,
                                            **kw)
     self.base = base
     self.augment = augment
     self.ad = {
         v: augment.D[k] if k in augment.D else 0
         for k, v in base.D.items()
     }
     valval = np.zeros((max(self.ad.keys()) + 1, ), dtype="int32")
     for i in range(valval.shape[0]):
         valval[i] = self.ad[i] if i in self.ad else 0
     self.adb = Val(valval)
Ejemplo n.º 20
0
 def __init__(self,
              indim=50,
              outdim=50,
              window=5,
              border_mode="half",
              stride=1,
              filter_flip=True,
              **kw):
     super(Conv1D, self).__init__(**kw)
     if isinstance(border_mode, tuple):
         (border_mode, ) = border_mode
     if isinstance(border_mode, int):
         border_mode = (border_mode, 0)
     self.border_mode = border_mode
     self.stride = stride
     self.filter_flip = filter_flip
     self.filter_shape = (outdim, indim, window, 1)
     self.filter = param(self.filter_shape, name="conv_w").glorotuniform()
     self.maskfilter_shape = (1, 1, window, 1)
     self.maskfilter = Val(np.ones(self.maskfilter_shape, dtype="float32"))
Ejemplo n.º 21
0
    def test_mask_propagation_all_states(self):
        m = SeqEncoder(VectorEmbed(maskid=0, indim=100, dim=7),
                       GRU(dim=7, innerdim=30)).all_outputs()\
            .maskoptions(MaskSetMode.ZERO)
        data = np.random.randint(1, 100, (5, 3), dtype="int32")
        ndata = np.zeros_like(data)
        data = np.concatenate([data, ndata], axis=1)

        dataval = Val(data)
        embvar = m.embedder(dataval)
        embpred = embvar.eval()
        embmaskpred = embvar.mask.eval()

        encvar = m(dataval)
        encpred = encvar.eval()
        encmaskpred = encvar.mask.eval()
        print encpred.shape
        print encmaskpred.shape
        print encmaskpred
        self.assertTrue(np.sum(encmaskpred - embmaskpred) == 0)
Ejemplo n.º 22
0
    def __init__(self, wordemb, wdic, **kw):
        D = wordemb.D
        nativeraretoken = wordemb.raretoken
        super(AdaptedWordEmb, self).__init__(worddic=wdic,
                                             value=False,
                                             dim=wordemb.outdim,
                                             normalize=wordemb.normalize,
                                             trainfrac=wordemb.trainfrac,
                                             raretoken=nativeraretoken,
                                             **kw)
        self.inner = wordemb

        self.ad = {
            v: D[k] if k in D else D[nativeraretoken]
            for k, v in wdic.items()
        }

        valval = np.zeros((max(self.ad.keys()) + 1, ), dtype="int32")
        for i in range(valval.shape[0]):
            valval[i] = self.ad[i] if i in self.ad else 0
        self.adb = Val(valval)
Ejemplo n.º 23
0
 def setvalue(self, v):
     if self.trainfrac == 0.0:
         self.W = Val(v, name="embedder_val")
     else:
         self.W = Parameter(v, lrmul=self.trainfrac, name="embedder")
Ejemplo n.º 24
0
 def __init__(self, entmat):
     self.em = Val(entmat)  # entmat: idx[word]^(numents, len(ent.name))
Ejemplo n.º 25
0
 def test_dimswap(self):
     xval = np.random.randint(0, 5, (2, 3, 4))
     x = Val(xval)
     y = x.dimswap(1, 0)
     eyval = xval.transpose(1, 0, 2)
     self.assertTrue(np.allclose(eyval, y.eval()))
Ejemplo n.º 26
0
from teafacto.core.base import tensorops as T, Val, param
import numpy as np
import sys

x = Val(np.random.random((10, 10)))
#y = Val(np.random.random((10,10)))
y = param((10, 10), name="y").uniform()
w = param((10, 10), name="w").uniform()

#z = T.dot(x, y)
z = (x + y)
u = z * w
s = T.nnet.sigmoid
s2 = T.nnet.sigmoid
print s == s2
sys.exit()
print z.allparams
print T.dot
print z.ndim
print z.dimswap
zd = z.dimswap(1, 0)
print z.dimswap(0, 1).allparams
print y.dimswap(0, 1).allparams
print T.nnet.conv.conv2d
print u.norm(2).allparams
print u.dimswap(0, 1).allparams
print T.nnet.softmax(z).allparams
zs = T.nnet.sigmoid(z)
zs = zs + x
zs.autobuild()
zs.autobuild()
Ejemplo n.º 27
0
 def __init__(self, vocsize, **kw):
     super(IdxToOneHot, self).__init__(vocsize, vocsize, **kw)
     self.W = Val(np.eye(vocsize, vocsize))
Ejemplo n.º 28
0
 def load(self, *data):
     self.data = [
         Val(d) if not isinstance(d, (Var, Val)) else d for d in data
     ]
     self.innervar = self.block(*self.data)
Ejemplo n.º 29
0
 def __init__(self, wordmat):
     self.em = Val(wordmat)
Ejemplo n.º 30
0
 def __init__(self, entmat):
     self.em = Val(entmat)