def setUp(self):
vocsize = 10
innerdim = 50
encdim = 30
seqlen = 5
batsize = 77
self.att = Attention(
LinearSumAttentionGenerator(indim=innerdim + encdim),
WeightedSumAttCon())
self.decwatt = SeqDecoder([
IdxToOneHot(vocsize),
GRU(dim=vocsize + encdim, innerdim=innerdim)
],
inconcat=True,
attention=self.att,
innerdim=innerdim)
self.decwoatt = SeqDecoder([
IdxToOneHot(vocsize),
GRU(dim=vocsize + encdim, innerdim=innerdim)
],
inconcat=True,
innerdim=innerdim)
self.attdata = np.random.random(
(batsize, seqlen, encdim)).astype("float32")
self.data = np.random.random((batsize, encdim)).astype("float32")
self.seqdata = np.random.randint(0, vocsize, (batsize, seqlen))
self.predshape = (batsize, seqlen, vocsize)
def init(self):
#memory
wencpg = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload, self.memdata, indim=self.outdim, outdim=self.encinnerdim+self.entembdim)
#encoder
wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath)
self.enc = SeqEncoder(
wencpg2,
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
#decoder
entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(self.memaddr(self.memblock, indim=self.decinnerdim, memdim=self.memblock.outdim, attdim=self.attdim), Softmax())
self.dec = SeqDecoder(
[entemb2, #self.memblock,
GRU(dim=entemb.outdim + self.encinnerdim, innerdim=self.decinnerdim), # GRU(dim=self.memblock.outdim + self.encinnerdim, innerdim=self.decinnerdim),
],
inconcat=True,
innerdim=self.decinnerdim,
softmaxoutblock=self.softmaxoutblock
)
def __init__(self,
wordembdim=50,
entembdim=200,
innerdim=200,
attdim=100,
outdim=1e4,
numwords=4e5,
**kw):
super(FBSeqSimpEncDecAtt, self).__init__(**kw)
self.indim = wordembdim
self.outdim = outdim
self.wordembdim = wordembdim
self.encinnerdim = innerdim
self.decinnerdim = innerdim
self.entembdim = entembdim
self.wordencoder = WordEmbed(indim=numwords,
outdim=self.wordembdim,
trainfrac=1.0)
self.rnn = RecStack(
self.wordencoder,
GRU(dim=self.wordembdim, innerdim=self.encinnerdim))
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
attdim=attdim)
attcon = WeightedSumAttCon()
self.dec = SeqDecoder([
VectorEmbed(indim=self.outdim, dim=self.entembdim),
GRU(dim=self.entembdim, innerdim=self.decinnerdim)
],
attention=Attention(attgen, attcon),
outconcat=True,
inconcat=False,
innerdim=self.encinnerdim + self.decinnerdim)
def test_vector_out(self):
decdim = 50
outvocsize = 17
outemb = IdxToOneHot(outvocsize)
outembdim = outvocsize
decrnus = [GRU(dim=outvocsize, innerdim=decdim)]
dec = SeqDecoder([outemb]+decrnus, innerdim=decdim*2, outconcat=True, inconcat=False)
ctxdata = np.random.random((1, decdim)).astype("float32")
seqdata = np.asarray([[2, 3, 4]])
pred = dec.predict(ctxdata, seqdata)
self.assertEqual(pred.shape, (1, 3, outvocsize))
dec = SeqDecoder([outemb]+decrnus, innerdim=decdim*2, softmaxoutblock=False, outconcat=True, inconcat=False)
pred = dec.predict(ctxdata, seqdata)
self.assertEqual(pred.shape, (1, 3, decdim*2))
def _getdecoder(self,
outvocsize=None,
outembdim=None,
outemb=None,
maskid=-1,
attention=None,
lastencinnerdim=None,
decinnerdim=None,
inconcat=False,
outconcat=True,
softmaxout=None,
dropout=None,
rnu=None):
lastencinnerdim = self.lastencinnerdim if lastencinnerdim is None else lastencinnerdim
decinnerdim = self.decinnerdim if decinnerdim is None else decinnerdim
rnu = GRU if rnu is None else rnu
dec = SeqDecoder.RNN(
emb=outemb,
embdim=outembdim,
embsize=outvocsize,
maskid=maskid,
ctxdim=lastencinnerdim,
attention=attention,
innerdim=decinnerdim,
inconcat=inconcat,
softmaxoutblock=softmaxout,
outconcat=outconcat,
dropout=dropout,
rnu=rnu,
dropout_h=dropout,
)
return dec
def __init__(self,
enclayers,
declayers,
attgen,
attcon,
decinnerdim,
statetrans=None,
vecout=False,
inconcat=True,
outconcat=False,
**kw):
enc = SeqEncoder(*enclayers)\
.with_outputs()\
.with_mask()\
.maskoptions(-1, MaskMode.AUTO, MaskSetMode.ZERO)
smo = False if vecout else None
dec = SeqDecoder(declayers,
attention=Attention(attgen, attcon),
innerdim=decinnerdim,
inconcat=inconcat,
softmaxoutblock=smo,
outconcat=outconcat)
super(SeqEncDecAtt, self).__init__(enc,
dec,
statetrans=statetrans,
**kw)
def setUp(self):
self.batsize = 70
self.vocsize = 27
self.encdim = 44
self.hdim = 33
self.statedim = 50
self.seqlen = 30
self.encodings_data = np.random.random(
(self.batsize, self.encdim)).astype("float32")
self.sequence_data = np.random.randint(0, self.vocsize,
(self.batsize, self.seqlen))
self.dec = SeqDecoder([
IdxToOneHot(self.vocsize),
GRU(dim=self.vocsize + self.encdim, innerdim=self.hdim),
LSTM(dim=self.hdim, innerdim=self.statedim)
],
inconcat=True,
innerdim=self.statedim)
class TestAttentionRNNDecoder(TestCase):
def setUp(self):
vocsize = 10
innerdim = 50
encdim = 30
seqlen = 5
batsize = 77
self.att = Attention(AttGen(BilinearDistance(innerdim, encdim)),
WeightedSumAttCon())
self.decwatt = SeqDecoder(
[IdxToOneHot(vocsize), GRU(dim=vocsize+encdim, innerdim=innerdim)],
inconcat=True,
attention=self.att,
innerdim=innerdim
)
self.decwoatt = SeqDecoder(
[IdxToOneHot(vocsize), GRU(dim=vocsize+encdim, innerdim=innerdim)],
inconcat=True,
innerdim=innerdim
)
self.attdata = np.random.random((batsize, seqlen, encdim)).astype("float32")
self.data = np.random.random((batsize, encdim)).astype("float32")
self.seqdata = np.random.randint(0, vocsize, (batsize, seqlen))
self.predshape = (batsize, seqlen, vocsize)
def test_shape(self):
pred = self.decwatt.predict(self.attdata, self.seqdata)
self.assertEqual(pred.shape, self.predshape)
def test_shape_wo_att(self):
pred = self.decwoatt.predict(self.data, self.seqdata)
self.assertEqual(pred.shape, self.predshape)
def test_attentiongenerator_param_in_allparams(self):
inps, outps = self.decwatt.autobuild(self.attdata, self.seqdata)
allparams = outps[0].allparams
self.assertIn(self.att.attentiongenerator.dist.W, allparams)
def test_attentiongenerator_param_not_in_params_of_dec_wo_att(self):
_, outps = self.decwoatt.autobuild(self.data, self.seqdata)
allparams = outps[0].allparams
self.assertNotIn(self.att.attentiongenerator.dist.W, allparams)
def setUp(self):
vocsize = 10
innerdim = 50
encdim = 30
seqlen = 5
batsize = 77
self.att = Attention(LinearSumAttentionGenerator(indim=innerdim + encdim), WeightedSumAttCon())
self.decwatt = SeqDecoder(
[IdxToOneHot(vocsize), GRU(dim=vocsize+encdim, innerdim=innerdim)],
inconcat=True,
attention=self.att,
innerdim=innerdim
)
self.decwoatt = SeqDecoder(
[IdxToOneHot(vocsize), GRU(dim=vocsize+encdim, innerdim=innerdim)],
inconcat=True,
innerdim=innerdim
)
self.attdata = np.random.random((batsize, seqlen, encdim)).astype("float32")
self.data = np.random.random((batsize, encdim)).astype("float32")
self.seqdata = np.random.randint(0, vocsize, (batsize, seqlen))
self.predshape = (batsize, seqlen, vocsize)
def init(self):
#MEMORY: encodes how entity is written + custom entity embeddings
wencpg = WordEncoderPlusGlove(numchars=self.numchars,
numwords=self.numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload,
self.memdata,
indim=self.outdim,
outdim=self.encinnerdim + self.entembdim)
#ENCODER: uses the same language encoder as memory
#wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath)
self.enc = RecStack(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
#ATTENTION
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
innerdim=self.attdim)
attcon = WeightedSumAttCon()
#DECODER
#entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(
self.memaddr(self.memblock,
indim=self.decinnerdim + self.encinnerdim,
memdim=self.memblock.outdim,
attdim=self.attdim), Softmax())
self.dec = SeqDecoder([
self.memblock,
GRU(dim=self.entembdim + self.encinnerdim,
innerdim=self.decinnerdim)
],
outconcat=True,
inconcat=False,
attention=Attention(attgen, attcon),
innerdim=self.decinnerdim + self.encinnerdim,
softmaxoutblock=self.softmaxoutblock)
def setUp(self):
self.batsize = 70
self.vocsize = 27
self.encdim = 44
self.hdim = 33
self.statedim = 50
self.seqlen = 30
self.encodings_data = np.random.random((self.batsize, self.encdim)).astype("float32")
self.sequence_data = np.random.randint(0, self.vocsize, (self.batsize, self.seqlen))
self.dec = SeqDecoder(
[IdxToOneHot(self.vocsize), GRU(dim=self.vocsize+self.encdim, innerdim=self.hdim), LSTM(dim=self.hdim, innerdim=self.statedim)],
inconcat=True,
innerdim=self.statedim
)
class TestSimpleRNNDecoder(TestCase):
def setUp(self):
self.batsize = 70
self.vocsize = 27
self.encdim = 44
self.hdim = 33
self.statedim = 50
self.seqlen = 30
self.encodings_data = np.random.random((self.batsize, self.encdim)).astype("float32")
self.sequence_data = np.random.randint(0, self.vocsize, (self.batsize, self.seqlen))
self.dec = SeqDecoder(
[IdxToOneHot(self.vocsize), GRU(dim=self.vocsize+self.encdim, innerdim=self.hdim), LSTM(dim=self.hdim, innerdim=self.statedim)],
inconcat=True,
innerdim=self.statedim
)
def test_rnndecoder_output_shape(self):
outvals = self.dec.predict(self.encodings_data, self.sequence_data)
self.assertEqual(outvals.shape, (self.batsize, self.seqlen, self.vocsize))
def __init__(self,
wordembdim=50,
wordencdim=50,
entembdim=200,
innerdim=200,
attdim=100,
outdim=1e4,
numwords=4e5,
numchars=128,
glovepath=None,
**kw):
super(FBSeqCompEncDecAtt, self).__init__(**kw)
self.indim = wordembdim + wordencdim
self.outdim = outdim
self.wordembdim = wordembdim
self.wordencdim = wordencdim
self.encinnerdim = innerdim
self.entembdim = entembdim
self.decinnerdim = innerdim
self.wordencoder = WordEncoderPlusGlove(numchars=numchars,
numwords=numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=glovepath)
self.rnn = RecStack(
self.wordencoder,
GRU(dim=wordembdim + wordencdim, innerdim=self.encinnerdim))
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
innerdim=attdim)
attcon = WeightedSumAttCon()
self.dec = SeqDecoder([
VectorEmbed(indim=self.outdim, dim=self.entembdim),
GRU(dim=self.entembdim, innerdim=self.decinnerdim)
],
attention=Attention(attgen, attcon),
outconcat=True,
inconcat=False,
innerdim=self.encinnerdim + self.decinnerdim)
def __init__(self, wordembdim=50, wordencdim=100, entembdim=200, innerdim=200, outdim=1e4, numwords=4e5, numchars=128, glovepath=None, **kw):
super(FBSeqCompositeEncDec, self).__init__(**kw)
self.indim = wordembdim + wordencdim
self.outdim = outdim
self.wordembdim = wordembdim
self.wordencdim = wordencdim
self.encinnerdim = innerdim
self.entembdim = entembdim
self.decinnerdim = innerdim
self.enc = SeqEncoder(
WordEncoderPlusGlove(numchars=numchars, numwords=numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath),
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
self.dec = SeqDecoder(
[VectorEmbed(indim=self.outdim, dim=self.entembdim), GRU(dim=self.entembdim+self.encinnerdim, innerdim=self.decinnerdim)],
inconcat=True,
innerdim=self.decinnerdim,
)
def __init__(self,
encdim=44,
invocsize=500,
outvocsize=27,
innerdim=300,
seqlen=20,
**kw):
super(idx2seq, self).__init__(**kw)
self.invocsize = invocsize
self.outvocsize = outvocsize
self.innerdim = innerdim
self.seqlen = seqlen
self.encdim = encdim
self.emb = VectorEmbed(indim=self.invocsize,
dim=self.encdim,
normalize=False)
self.dec = SeqDecoder([
IdxToOneHot(self.outvocsize),
GRU(dim=self.outvocsize + self.encdim,
innerdim=self.innerdim,
nobias=True)
],
inconcat=True,
innerdim=self.innerdim)
class TestSimpleRNNDecoder(TestCase):
def setUp(self):
self.batsize = 70
self.vocsize = 27
self.encdim = 44
self.hdim = 33
self.statedim = 50
self.seqlen = 30
self.encodings_data = np.random.random(
(self.batsize, self.encdim)).astype("float32")
self.sequence_data = np.random.randint(0, self.vocsize,
(self.batsize, self.seqlen))
self.dec = SeqDecoder([
IdxToOneHot(self.vocsize),
GRU(dim=self.vocsize + self.encdim, innerdim=self.hdim),
LSTM(dim=self.hdim, innerdim=self.statedim)
],
inconcat=True,
innerdim=self.statedim)
def test_rnndecoder_output_shape(self):
outvals = self.dec.predict(self.encodings_data, self.sequence_data)
self.assertEqual(outvals.shape,
(self.batsize, self.seqlen, self.vocsize))
class TestAttentionRNNDecoder(TestCase):
def setUp(self):
vocsize = 10
innerdim = 50
encdim = 30
seqlen = 5
batsize = 77
self.att = Attention(LinearSumAttentionGenerator(indim=innerdim + encdim), WeightedSumAttCon())
self.decwatt = SeqDecoder(
[IdxToOneHot(vocsize), GRU(dim=vocsize+encdim, innerdim=innerdim)],
inconcat=True,
attention=self.att,
innerdim=innerdim
)
self.decwoatt = SeqDecoder(
[IdxToOneHot(vocsize), GRU(dim=vocsize+encdim, innerdim=innerdim)],
inconcat=True,
innerdim=innerdim
)
self.attdata = np.random.random((batsize, seqlen, encdim)).astype("float32")
self.data = np.random.random((batsize, encdim)).astype("float32")
self.seqdata = np.random.randint(0, vocsize, (batsize, seqlen))
self.predshape = (batsize, seqlen, vocsize)
def test_shape(self):
pred = self.decwatt.predict(self.attdata, self.seqdata)
self.assertEqual(pred.shape, self.predshape)
def test_shape_wo_att(self):
pred = self.decwoatt.predict(self.data, self.seqdata)
self.assertEqual(pred.shape, self.predshape)
def test_attentiongenerator_param_in_allparams(self):
self.decwatt.predict(self.attdata, self.seqdata)
allparams = self.decwatt.output.allparams
self.assertIn(self.att.attentiongenerator.W, allparams)
def test_attentiongenerator_param_not_in_params_of_dec_wo_att(self):
self.decwoatt.predict(self.data, self.seqdata)
allparams = self.decwoatt.output.allparams
self.assertNotIn(self.att.attentiongenerator.W, allparams)
def test_vector_out(self):
decdim = 50
outvocsize = 17
outemb = IdxToOneHot(outvocsize)
outembdim = outvocsize
decrnus = [GRU(dim=outvocsize, innerdim=decdim)]
dec = SeqDecoder([outemb] + decrnus,
innerdim=decdim * 2,
outconcat=True,
inconcat=False)
ctxdata = np.random.random((2, decdim)).astype("float32")
seqdata = np.asarray([[2, 3, 4], [2, 3, 4]])
pred = dec.predict(ctxdata, seqdata)
self.assertEqual(pred.shape, (2, 3, outvocsize))
dec = SeqDecoder([outemb] + decrnus,
innerdim=decdim * 2,
softmaxoutblock=False,
outconcat=True,
inconcat=False)
pred = dec.predict(ctxdata, seqdata)
self.assertEqual(pred.shape, (2, 3, decdim * 2))
