def __init__(self,
wordembdim=50,
entembdim=200,
innerdim=200,
attdim=100,
outdim=1e4,
numwords=4e5,
**kw):
super(FBSeqSimpEncDecAtt, self).__init__(**kw)
self.indim = wordembdim
self.outdim = outdim
self.wordembdim = wordembdim
self.encinnerdim = innerdim
self.decinnerdim = innerdim
self.entembdim = entembdim
self.wordencoder = WordEmbed(indim=numwords,
outdim=self.wordembdim,
trainfrac=1.0)
self.rnn = RecStack(
self.wordencoder,
GRU(dim=self.wordembdim, innerdim=self.encinnerdim))
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
attdim=attdim)
attcon = WeightedSumAttCon()
self.dec = SeqDecoder([
VectorEmbed(indim=self.outdim, dim=self.entembdim),
GRU(dim=self.entembdim, innerdim=self.decinnerdim)
],
attention=Attention(attgen, attcon),
outconcat=True,
inconcat=False,
innerdim=self.encinnerdim + self.decinnerdim)
def init(self):
#memory
wencpg = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload, self.memdata, indim=self.outdim, outdim=self.encinnerdim+self.entembdim)
#encoder
wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath)
self.enc = SeqEncoder(
wencpg2,
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
#decoder
entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(self.memaddr(self.memblock, indim=self.decinnerdim, memdim=self.memblock.outdim, attdim=self.attdim), Softmax())
self.dec = SeqDecoder(
[entemb2, #self.memblock,
GRU(dim=entemb.outdim + self.encinnerdim, innerdim=self.decinnerdim), # GRU(dim=self.memblock.outdim + self.encinnerdim, innerdim=self.decinnerdim),
],
inconcat=True,
innerdim=self.decinnerdim,
softmaxoutblock=self.softmaxoutblock
)
def test_get_params(self):
gru = GRU(innerdim=100, dim=20)
params = {
gru.um, gru.wm, gru.uhf, gru.whf, gru.u, gru.w, gru.bm, gru.bhf,
gru.b
}
self.assertEqual(params, gru.get_params())
def setUp(self):
vocsize = 10
innerdim = 50
encdim = 30
seqlen = 5
batsize = 77
self.att = Attention(
LinearSumAttentionGenerator(indim=innerdim + encdim),
WeightedSumAttCon())
self.decwatt = SeqDecoder([
IdxToOneHot(vocsize),
GRU(dim=vocsize + encdim, innerdim=innerdim)
],
inconcat=True,
attention=self.att,
innerdim=innerdim)
self.decwoatt = SeqDecoder([
IdxToOneHot(vocsize),
GRU(dim=vocsize + encdim, innerdim=innerdim)
],
inconcat=True,
innerdim=innerdim)
self.attdata = np.random.random(
(batsize, seqlen, encdim)).astype("float32")
self.data = np.random.random((batsize, encdim)).astype("float32")
self.seqdata = np.random.randint(0, vocsize, (batsize, seqlen))
self.predshape = (batsize, seqlen, vocsize)
def __init__(self, innerdim=50, input_vocsize=100, output_vocsize=100, **kw):
super(SimpleEncoderDecoder, self).__init__(**kw)
input_embedder = IdxToOneHot(input_vocsize)
output_embedder = IdxToOneHot(output_vocsize)
encrec = GRU(dim=input_vocsize, innerdim=innerdim)
decrecrnu = GRU(dim=output_vocsize, innerdim=innerdim)
self.enc = SeqEncoder(input_embedder, encrec)
self.dec = SeqDecoder([output_embedder, decrecrnu], outconcat=True, inconcat=False, innerdim=innerdim+innerdim)
def test_output_shape(self):
indim = 20
innerdim = 50
batsize = 200
seqlen = 5
data = np.random.random((batsize, seqlen, indim)).astype("float32")
gru = GRU(innerdim=innerdim, dim=indim)
grupred = gru.predict(data)
self.assertEqual(grupred.shape, (batsize, seqlen, innerdim))
def test_output_shape(self):
indim = 20
innerdim = 50
batsize = 200
seqlen = 5
data = np.random.random((batsize, seqlen, indim)).astype("float32")
gru = GRU(innerdim=innerdim, dim=indim)
grupred = gru.predict(data)
self.assertEqual(grupred.shape, (batsize, seqlen, innerdim))
def __init__(self, vocsize=25, outvocsize=25, encdim=200, innerdim=200, attdim=50, **kw):
super(RewAttSumDecoder, self).__init__(**kw)
self.rnn = SeqEncoder(IdxToOneHot(vocsize), GRU(dim=vocsize, innerdim=encdim)).all_outputs
attgen = LinearGateAttentionGenerator(indim=innerdim+encdim, innerdim=attdim)
attcon = WeightedSumAttCon()
self.dec = SeqDecoder([IdxToOneHot(outvocsize), GRU(dim=outvocsize+encdim, innerdim=innerdim)],
inconcat=True,
attention=Attention(attgen, attcon),
innerdim=innerdim)
def __init__(self, vocsize=25, outvocsize=20, encdim=200, innerdim=200, attdim=50, **kw):
super(FwdAttRNNEncDecoder, self).__init__(**kw)
self.emb = IdxToOneHot(vocsize)
attgen = LinearGateAttentionGenerator(indim=innerdim+vocsize, innerdim=attdim)
attcon = SeqEncoder(None,
GRU(dim=vocsize, innerdim=encdim))
self.dec = SeqDecoder([IdxToOneHot(outvocsize), GRU(dim=outvocsize, innerdim=innerdim)],
outconcat=True, inconcat=False,
attention=Attention(attgen, attcon),
innerdim=innerdim+encdim)
def __init__(self, vocsize=25, encdim=200, innerdim=200, seqlen=50, **kw):
super(RNNAutoEncoder, self).__init__(**kw)
self.seqlen = seqlen
self.encoder = SeqEncoder(
IdxToOneHot(vocsize=vocsize),
GRU(dim=vocsize, innerdim=encdim))
self.decoder = SeqDecoder([IdxToOneHot(vocsize), GRU(dim=vocsize+encdim, innerdim=innerdim)],
innerdim=innerdim,
inconcat=True
)
def test_if_prediction_is_equivalent_to_manually_constructed_theano_graph(self):
indim = 20
innerdim = 50
batsize = 200
seqlen = 5
data = np.random.random((batsize, seqlen, indim)).astype("float32")
gru = GRU(innerdim=innerdim, dim=indim)
grupred = gru.predict(data)[:, -1, :]
tgru_in, tgru_out = self.build_theano_gru(innerdim, indim, batsize, gru)
tgrupred = tgru_out.eval({tgru_in: data.astype("float32")})
print np.sum(np.abs(tgrupred-grupred))
self.assertTrue(np.allclose(grupred, tgrupred))
def init(self):
#MEMORY: encodes how entity is written + custom entity embeddings
wencpg = WordEncoderPlusGlove(numchars=self.numchars,
numwords=self.numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload,
self.memdata,
indim=self.outdim,
outdim=self.encinnerdim + self.entembdim)
#ENCODER: uses the same language encoder as memory
#wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath)
self.enc = RecStack(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
#ATTENTION
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
innerdim=self.attdim)
attcon = WeightedSumAttCon()
#DECODER
#entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(
self.memaddr(self.memblock,
indim=self.decinnerdim + self.encinnerdim,
memdim=self.memblock.outdim,
attdim=self.attdim), Softmax())
self.dec = SeqDecoder([
self.memblock,
GRU(dim=self.entembdim + self.encinnerdim,
innerdim=self.decinnerdim)
],
outconcat=True,
inconcat=False,
attention=Attention(attgen, attcon),
innerdim=self.decinnerdim + self.encinnerdim,
softmaxoutblock=self.softmaxoutblock)
def setUp(self):
batsize = 1000
seqlen = 19
indim = 71
hdim = 51
hdim2 = 61
self.outdim = 47
self.enc = SeqEncoder(None,
GRU(dim=indim, innerdim=hdim),
GRU(dim=hdim, innerdim=hdim2),
GRU(dim=hdim2, innerdim=self.outdim))
self.enc = self.doswitches(self.enc)
self.data = np.random.random((batsize, seqlen, indim)).astype("float32")
self.p = self.enc.predict
self.out = self.p(self.data)
def __init__(self,
wordembdim=50,
wordencdim=100,
innerdim=200,
outdim=1e4,
numwords=4e5,
numchars=128,
glovepath=None,
**kw):
super(FBBasicCompositeEncoder, self).__init__(**kw)
self.indim = wordembdim + wordencdim
self.outdim = outdim
self.wordembdim = wordembdim
self.wordencdim = wordencdim
self.innerdim = innerdim
self.enc = SeqEncoder(
WordEncoderPlusGlove(numchars=numchars,
numwords=numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=glovepath),
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.innerdim))
self.out = Lin(indim=self.innerdim, dim=self.outdim)
def test_gru_with_mask(self):
indim = 2
innerdim = 5
batsize = 4
seqlen = 3
data = np.random.random((batsize, seqlen, indim)).astype("float32")
mask = np.zeros((batsize, seqlen)).astype("float32")
mask[:, 0] = 1.
mask[0, :] = 1.
gru = GRU(innerdim=innerdim, dim=indim)
grupred = gru.predict(data, mask)
print grupred
self.assertEqual(grupred.shape, (batsize, seqlen, innerdim))
#self.assertTrue(np.allclose(grupred[1:, 1:, :], np.zeros_like(grupred[1:, 1:, :])))
self.assertTrue(np.all(abs(grupred[0, ...]) > 0))
self.assertTrue(np.all(abs(grupred[:, 0, :]) > 0))
def test_gru_with_mask(self):
indim = 2
innerdim = 5
batsize = 4
seqlen = 3
data = np.random.random((batsize, seqlen, indim)).astype("float32")
mask = np.zeros((batsize, seqlen)).astype("float32")
mask[:, 0] = 1.
mask[0, :] = 1.
gru = GRU(innerdim=innerdim, dim=indim)
grupred = gru.predict(data, mask)
print grupred
self.assertEqual(grupred.shape, (batsize, seqlen, innerdim))
#self.assertTrue(np.allclose(grupred[1:, 1:, :], np.zeros_like(grupred[1:, 1:, :])))
self.assertTrue(np.all(abs(grupred[0, ...]) > 0))
self.assertTrue(np.all(abs(grupred[:, 0, :]) > 0))
def __init__(self, invocsize=27, outvocsize=500, innerdim=300, **kw):
super(seq2idx, self).__init__(**kw)
self.invocsize = invocsize
self.outvocsize = outvocsize
self.innerdim = innerdim
self.enc = SeqEncoder(
VectorEmbed(indim=self.invocsize, dim=self.invocsize),
GRU(dim=self.invocsize, innerdim=self.innerdim))
self.outlin = Lin(indim=self.innerdim, dim=self.outvocsize)
def __init__(self,
indim=220,
outdim=200,
maskid=0,
**kw): # indim is number of characters
super(WordEncoder, self).__init__(**kw)
self.enc = SeqEncoder(IdxToOneHot(indim),
GRU(dim=indim, innerdim=outdim)).maskoptions(
maskid, MaskMode.AUTO)
def test_output_shape(self):
batsize = 100
seqlen = 5
dim = 50
indim = 13
m = SeqEncoder(IdxToOneHot(13), GRU(dim=indim, innerdim=dim))
data = np.random.randint(0, indim, (batsize, seqlen)).astype("int32")
mpred = m.predict(data)
self.assertEqual(mpred.shape, (batsize, dim))
def test_memory_block_with_seq_encoder_dynamic_fail(self):
invocabsize = 5
encdim = 13
gru = GRU(dim=invocabsize, innerdim=encdim)
payload = SeqEncoder(IdxToOneHot(vocsize=invocabsize), gru)
dynmemb = MemoryBlock(payload, outdim=encdim)
idxs = [0, 2, 5]
#dynmemb.predict(idxs)
self.assertRaises(AssertionError, lambda: dynmemb.predict(idxs))
def setUp(self):
self.dims = [50, 20, 30, 40]
grus = [
GRU(dim=self.dims[i], innerdim=self.dims[i + 1])
for i in range(len(self.dims) - 1)
]
self.s = stack(*grus)
self.paramnames = [
"um", "wm", "uhf", "whf", "u", "w", "bm", "bhf", "b"
]
def setUp(self):
dim = 50
self.outdim = 100
batsize = 1000
seqlen = 19
self.enc = SeqEncoder(None, GRU(dim=dim, innerdim=self.outdim))
self.enc = self.doswitches(self.enc)
self.data = np.random.random((batsize, seqlen, dim)).astype("float32")
self.p = self.enc.predict
self.out = self.p(self.data)
def __init__(self,
wordembdim=50,
wordencdim=50,
entembdim=200,
innerdim=200,
attdim=100,
outdim=1e4,
numwords=4e5,
numchars=128,
glovepath=None,
**kw):
super(FBSeqCompEncDecAtt, self).__init__(**kw)
self.indim = wordembdim + wordencdim
self.outdim = outdim
self.wordembdim = wordembdim
self.wordencdim = wordencdim
self.encinnerdim = innerdim
self.entembdim = entembdim
self.decinnerdim = innerdim
self.wordencoder = WordEncoderPlusGlove(numchars=numchars,
numwords=numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=glovepath)
self.rnn = RecStack(
self.wordencoder,
GRU(dim=wordembdim + wordencdim, innerdim=self.encinnerdim))
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
innerdim=attdim)
attcon = WeightedSumAttCon()
self.dec = SeqDecoder([
VectorEmbed(indim=self.outdim, dim=self.entembdim),
GRU(dim=self.entembdim, innerdim=self.decinnerdim)
],
attention=Attention(attgen, attcon),
outconcat=True,
inconcat=False,
innerdim=self.encinnerdim + self.decinnerdim)
def __init__(self, wordembdim=50, wordencdim=100, entembdim=200, innerdim=200, outdim=1e4, numwords=4e5, numchars=128, glovepath=None, **kw):
super(FBSeqCompositeEncDec, self).__init__(**kw)
self.indim = wordembdim + wordencdim
self.outdim = outdim
self.wordembdim = wordembdim
self.wordencdim = wordencdim
self.encinnerdim = innerdim
self.entembdim = entembdim
self.decinnerdim = innerdim
self.enc = SeqEncoder(
WordEncoderPlusGlove(numchars=numchars, numwords=numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath),
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
self.dec = SeqDecoder(
[VectorEmbed(indim=self.outdim, dim=self.entembdim), GRU(dim=self.entembdim+self.encinnerdim, innerdim=self.decinnerdim)],
inconcat=True,
innerdim=self.decinnerdim,
)
def __init__(
self,
entembdim=50,
wordembdim=50,
wordencdim=100,
memdata=None,
attdim=100,
numchars=128, # number of different chars
numwords=4e5, # number of different words
glovepath=None,
innerdim=100, # dim of memory payload encoder output
outdim=1e4, # number of entities
memaddr=DotMemAddr,
**kw):
super(FBMemMatch, self).__init__(**kw)
self.wordembdim = wordembdim
self.wordencdim = wordencdim
self.entembdim = entembdim
self.attdim = attdim
self.encinnerdim = innerdim
self.outdim = outdim
memaddr = TransDotMemAddr
# memory encoder per word
#wencpg = WordEmbed(indim=numwords, outdim=self.wordembdim, trainfrac=1.0)
wordencoder = WordEncoderPlusGlove(numchars=numchars,
numwords=numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=glovepath)
# memory encoder for one cell
self.phraseencoder = SeqEncoder(
wordencoder,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
# entity embedder
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.entembs = entemb(
memdata[0]) #Val(np.arange(0, self.outdim, dtype="int32")))
# memory block
self.mempayload = self.phraseencoder #ConcatBlock(entemb, self.phraseencoder)
self.memblock = MemoryBlock(
self.mempayload,
memdata[1],
indim=self.outdim,
outdim=self.encinnerdim) # + self.entembdim)
# memory addressing
self.mema = memaddr(self.memblock,
memdim=self.memblock.outdim,
attdim=attdim,
indim=self.encinnerdim)
def test_memory_block_with_seq_encoder_static_fail(self):
invocabsize = 5
memsize = 10
seqlen = 3
encdim = 13
data = np.random.randint(0, invocabsize, (memsize, seqlen))
gru = GRU(dim=invocabsize, innerdim=encdim)
payload = SeqEncoder(IdxToOneHot(vocsize=invocabsize), gru)
memb = MemoryBlock(payload, data, outdim=encdim)
idxs = [0, 2, 5]
self.assertRaises(AssertionError, lambda: memb.predict(idxs, data))
def test_output_shape_w_mask(self):
batsize = 2
seqlen = 5
dim = 3
indim = 7
m = SeqEncoder(IdxToOneHot(indim), GRU(dim=indim, innerdim=dim)).all_outputs
data = np.random.randint(0, indim, (batsize, seqlen)).astype("int32")
mask = np.zeros_like(data).astype("float32")
mask[:, 0:2] = 1
weights = np.ones_like(data).astype("float32")
mpred = m.predict(data, weights, mask)
self.assertEqual(mpred.shape, (batsize, seqlen, dim))
def test_recappl(self):
batsize = 100
self.dims = [50, 20, 30, 40]
recstack = RecStack(*[
GRU(dim=self.dims[i], innerdim=self.dims[i + 1])
for i in range(len(self.dims) - 1)
])
mu = RecPredictor(recstack).init(batsize)
for i in range(3):
inpval = np.random.random((batsize, 50)).astype("float32")
outpvals = mu.feed(inpval)
self.assertEqual(outpvals.shape, (batsize, self.dims[-1]))
def test_memory_block_with_seq_encoder(self):
invocabsize = 5
memsize = 10
seqlen = 3
encdim = 13
data = np.random.randint(0, invocabsize, (memsize, seqlen))
gru = GRU(dim=invocabsize, innerdim=encdim)
payload = SeqEncoder(IdxToOneHot(vocsize=invocabsize), gru)
memb = MemoryBlock(payload, data, indim=invocabsize, outdim=encdim)
idxs = [0, 2, 5]
memory_element = memb.predict(idxs)
self.assertEqual(memory_element.shape, (len(idxs), encdim))
gruparams = set([getattr(gru, pname) for pname in gru.paramnames])
allparams = set(memb.output.allparams)
self.assertEqual(gruparams.intersection(allparams), allparams)
def test_mask_dynamic_pad(self):
batsize = 10
seqlen = 5
dim = 6
indim = 5
m = SeqEncoder(IdxToOneHot(indim),
GRU(dim=indim,
innerdim=dim)).maskoption(-1).all_outputs()
data = np.random.randint(0, indim, (batsize, seqlen)).astype("int32")
rmasker = np.random.randint(2, seqlen, (batsize, )).astype("int32")
print rmasker
for i in range(data.shape[0]):
data[i, rmasker[i]:] = -1
print data
pred = m.predict(data)
print pred
def test_mask_no_state_updates(self):
batsize = 10
seqlen = 3
dim = 7
indim = 5
m = SeqEncoder(IdxToOneHot(indim), GRU(dim=indim, innerdim=dim)).maskoption(-1).all_outputs
data = np.random.randint(0, indim, (batsize, seqlen)).astype("int32")
data[:, 1] = 0
ndata = np.ones_like(data) * -1
data = np.concatenate([data, ndata], axis=1)
pred = m.predict(data)
for i in range(1, pred.shape[1]):
print np.linalg.norm(pred[:, i - 1, :] - pred[:, i, :])
if i < seqlen:
self.assertTrue(not np.allclose(pred[:, i - 1, :], pred[:, i, :]))
else:
self.assertTrue(np.allclose(pred[:, i - 1, :], pred[:, i, :]))
def setUp(self):
self.batsize = 70
self.vocsize = 27
self.encdim = 44
self.hdim = 33
self.statedim = 50
self.seqlen = 30
self.encodings_data = np.random.random(
(self.batsize, self.encdim)).astype("float32")
self.sequence_data = np.random.randint(0, self.vocsize,
(self.batsize, self.seqlen))
self.dec = SeqDecoder([
IdxToOneHot(self.vocsize),
GRU(dim=self.vocsize + self.encdim, innerdim=self.hdim),
LSTM(dim=self.hdim, innerdim=self.statedim)
],
inconcat=True,
innerdim=self.statedim)
