def test_output_shape(self):
batsize = 100
seqlen = 5
dim = 50
indim = 13
m = SeqEncoder(IdxToOneHot(13), GRU(dim=indim, innerdim=dim))
data = np.random.randint(0, indim, (batsize, seqlen)).astype("int32")
mpred = m.predict(data)
self.assertEqual(mpred.shape, (batsize, dim))
def test_output_shape_w_mask(self):
batsize = 2
seqlen = 5
dim = 3
indim = 7
m = SeqEncoder(IdxToOneHot(indim), GRU(dim=indim, innerdim=dim)).all_outputs
data = np.random.randint(0, indim, (batsize, seqlen)).astype("int32")
mask = np.zeros_like(data).astype("float32")
mask[:, 0:2] = 1
weights = np.ones_like(data).astype("float32")
mpred = m.predict(data, weights, mask)
self.assertEqual(mpred.shape, (batsize, seqlen, dim))
def test_mask_no_state_updates(self):
batsize = 10
seqlen = 3
dim = 7
indim = 5
m = SeqEncoder(IdxToOneHot(indim), GRU(dim=indim, innerdim=dim)).maskoption(-1).all_outputs
data = np.random.randint(0, indim, (batsize, seqlen)).astype("int32")
data[:, 1] = 0
ndata = np.ones_like(data) * -1
data = np.concatenate([data, ndata], axis=1)
pred = m.predict(data)
for i in range(1, pred.shape[1]):
print np.linalg.norm(pred[:, i - 1, :] - pred[:, i, :])
if i < seqlen:
self.assertTrue(not np.allclose(pred[:, i - 1, :], pred[:, i, :]))
else:
self.assertTrue(np.allclose(pred[:, i - 1, :], pred[:, i, :]))
def setUp(self):
dim = 50
self.outdim = 100
batsize = 1000
seqlen = 19
self.enc = SeqEncoder(None, GRU(dim=dim, innerdim=self.outdim))
self.enc = self.doswitches(self.enc)
self.data = np.random.random((batsize, seqlen, dim)).astype("float32")
self.out = self.enc.predict(self.data)
def test_mask_zero_mask_with_custom_maskid(self):
batsize = 10
seqlen = 3
dim = 7
indim = 5
m = SeqEncoder(IdxToOneHot(indim), GRU(dim=indim, innerdim=dim)).maskoptions(-1, MaskSetMode.ZERO).all_outputs
data = np.random.randint(0, indim, (batsize, seqlen)).astype("int32")
data[:, 1] = 0
ndata = np.ones_like(data) * -1
data = np.concatenate([data, ndata], axis=1)
pred = m.predict(data)
for i in range(pred.shape[1]):
print np.linalg.norm(pred[:, i - 1, :] - pred[:, i, :])
if i < seqlen:
for j in range(pred.shape[0]):
self.assertTrue(np.linalg.norm(pred[j, i, :]) > 0.0)
else:
for j in range(pred.shape[0]):
self.assertTrue(np.linalg.norm(pred[j, i, :]) == 0.0)
class SimpleRNNEncoderTest(TestCase):
expectedparams = ["um", "wm", "uhf", "whf", "u", "w", "bm", "bhf", "b"]
expectednumberparams = 1
def setUp(self):
dim = 50
self.outdim = 100
batsize = 1000
seqlen = 19
self.enc = SeqEncoder(None, GRU(dim=dim, innerdim=self.outdim))
self.enc = self.doswitches(self.enc)
self.data = np.random.random((batsize, seqlen, dim)).astype("float32")
self.out = self.enc.predict(self.data)
def test_output_shape(self):
self.assertEqual(self.out.shape, self.expectshape(self.data.shape, self.outdim))
def expectshape(self, datashape, outdim):
return (datashape[0], outdim)
def test_all_output_parameters(self):
outputs = self.enc.wrapply(*self.enc.inputs)
if issequence(outputs) and len(outputs) > 1:
outputparamsets = [x.allparams for x in outputs]
for i in range(len(outputparamsets)):
for j in range(i, len(outputparamsets)):
self.assertSetEqual(outputparamsets[i], outputparamsets[j])
if issequence(outputs):
outputs = outputs[0]
outputparamcounts = {}
for paramname in [x.name for x in outputs.allparams]:
if paramname not in outputparamcounts:
outputparamcounts[paramname] = 0
outputparamcounts[paramname] += 1
for (_, y) in outputparamcounts.items():
self.assertEqual(y, self.expectednumberparams)
self.assertSetEqual(set(outputparamcounts.keys()), set(self.expectedparams))
def doswitches(self, enc):
return enc
def setUp(self):
batsize = 1000
seqlen = 19
indim = 71
hdim = 51
hdim2 = 61
self.outdim = 47
self.enc = SeqEncoder(None,
GRU(dim=indim, innerdim=hdim),
GRU(dim=hdim, innerdim=hdim2),
GRU(dim=hdim2, innerdim=self.outdim))
self.enc = self.doswitches(self.enc)
self.data = np.random.random((batsize, seqlen, indim)).astype("float32")
self.out = self.enc.predict(self.data)
def test_memory_block_with_seq_encoder(self):
invocabsize = 5
memsize = 10
seqlen = 3
encdim = 13
data = np.random.randint(0, invocabsize, (memsize, seqlen))
gru = GRU(dim=invocabsize, innerdim=encdim)
payload = SeqEncoder(IdxToOneHot(vocsize=invocabsize), gru)
memb = MemoryBlock(payload, data, indim=invocabsize, outdim=encdim)
idxs = [0, 2, 5]
memory_element = memb.predict(idxs)
self.assertEqual(memory_element.shape, (len(idxs), encdim))
gruparams = set([getattr(gru, pname) for pname in gru.paramnames])
allparams = set(memb.output.allparams)
self.assertEqual(gruparams.intersection(allparams), allparams)
def __init__(self,
indim=400,
inpembdim=50,
inpemb=None,
mode="concat",
innerdim=100,
numouts=1,
maskid=0,
bidir=False,
maskmode=MaskMode.NONE,
**kw):
super(SimpleSeq2MultiVec, self).__init__(**kw)
if inpemb is None:
if inpembdim is None:
inpemb = IdxToOneHot(indim)
inpembdim = indim
else:
inpemb = VectorEmbed(indim=indim, dim=inpembdim)
elif inpemb is False:
inpemb = None
else:
inpembdim = inpemb.outdim
if not issequence(innerdim):
innerdim = [innerdim]
innerdim[-1] += numouts
rnn, lastdim = self.makernu(inpembdim, innerdim, bidir=bidir)
self.outdim = lastdim * numouts if mode == "concat" else lastdim
self.maskid = maskid
self.inpemb = inpemb
self.numouts = numouts
self.mode = mode
self.bidir = bidir
if not issequence(rnn):
rnn = [rnn]
self.enc = SeqEncoder(inpemb, *rnn).maskoptions(maskid, maskmode)
self.enc.all_outputs()
class SimpleRNNEncoderTest(TestCase):
expectedparams = ["um", "wm", "uhf", "whf", "u", "w", "bm", "bhf", "b"]
expectednumberparams = 1
def setUp(self):
dim = 50
self.outdim = 100
batsize = 1000
seqlen = 19
self.enc = SeqEncoder(None, GRU(dim=dim, innerdim=self.outdim))
self.enc = self.doswitches(self.enc)
self.data = np.random.random((batsize, seqlen, dim)).astype("float32")
self.p = self.enc.predict
self.out = self.p(self.data)
def test_output_shape(self):
out = self.out
if isinstance(self.out, tuple):
out = self.out[0]
self.assertEqual(out.shape, self.expectshape(self.data.shape, self.outdim))
def expectshape(self, datashape, outdim):
return (datashape[0], outdim)
def test_all_output_parameters(self):
outputs = self.enc.wrapply(*self.p.inps)
if issequence(outputs) and len(outputs) > 1:
outputparamsets = [x.allparams for x in outputs if isinstance(x, (Var, Val))]
for i in range(len(outputparamsets)):
for j in range(i, len(outputparamsets)):
self.assertSetEqual(outputparamsets[i], outputparamsets[j])
if issequence(outputs):
outputs = outputs[0]
outputparamcounts = {}
for paramname in [x.name for x in outputs.allparams]:
if paramname not in outputparamcounts:
outputparamcounts[paramname] = 0
outputparamcounts[paramname] += 1
for (_, y) in outputparamcounts.items():
self.assertEqual(y, self.expectednumberparams)
self.assertSetEqual(set(outputparamcounts.keys()), set(self.expectedparams))
def doswitches(self, enc):
return enc
def __init__(self, entembdim=50,
wordembdim=50,
wordencdim=100,
memdata=None,
attdim=100,
numchars=128, # number of different chars
numwords=4e5, # number of different words
glovepath=None,
innerdim=100, # dim of memory payload encoder output
outdim=1e4, # number of entities
memaddr=DotMemAddr, **kw):
super(FBMemMatch, self).__init__(**kw)
self.wordembdim = wordembdim
self.wordencdim = wordencdim
self.entembdim = entembdim
self.attdim = attdim
self.encinnerdim = innerdim
self.outdim = outdim
memaddr = TransDotMemAddr
# memory encoder per word
#wencpg = WordEmbed(indim=numwords, outdim=self.wordembdim, trainfrac=1.0)
wordencoder = WordEncoderPlusGlove(numchars=numchars, numwords=numwords, encdim=self.wordencdim,
embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath)
# memory encoder for one cell
self.phraseencoder = SeqEncoder(
wordencoder,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim)
)
# entity embedder
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.entembs = entemb(memdata[0]) #Val(np.arange(0, self.outdim, dtype="int32")))
# memory block
self.mempayload = self.phraseencoder #ConcatBlock(entemb, self.phraseencoder)
self.memblock = MemoryBlock(self.mempayload, memdata[1], indim=self.outdim,
outdim=self.encinnerdim)# + self.entembdim)
# memory addressing
self.mema = memaddr(self.memblock,
memdim=self.memblock.outdim, attdim=attdim, indim=self.encinnerdim)
def __init__(self, wordembdim=50, wordencdim=100, entembdim=200, innerdim=200, outdim=1e4, numwords=4e5, numchars=128, glovepath=None, **kw):
super(FBSeqCompositeEncDec, self).__init__(**kw)
self.indim = wordembdim + wordencdim
self.outdim = outdim
self.wordembdim = wordembdim
self.wordencdim = wordencdim
self.encinnerdim = innerdim
self.entembdim = entembdim
self.decinnerdim = innerdim
self.enc = SeqEncoder(
WordEncoderPlusGlove(numchars=numchars, numwords=numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath),
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
self.dec = SeqDecoder(
[VectorEmbed(indim=self.outdim, dim=self.entembdim), GRU(dim=self.entembdim+self.encinnerdim, innerdim=self.decinnerdim)],
inconcat=True,
innerdim=self.decinnerdim,
)
class StackRNNEncoderTest(SimpleRNNEncoderTest):
expectednumberparams = 3
def setUp(self):
batsize = 1000
seqlen = 19
indim = 71
hdim = 51
hdim2 = 61
self.outdim = 47
self.enc = SeqEncoder(None,
GRU(dim=indim, innerdim=hdim),
GRU(dim=hdim, innerdim=hdim2),
GRU(dim=hdim2, innerdim=self.outdim))
self.enc = self.doswitches(self.enc)
self.data = np.random.random((batsize, seqlen, indim)).astype("float32")
self.out = self.enc.predict(self.data)
def doswitches(self, enc):
return enc
def test_memory_block_with_seq_encoder_dynamic(self):
invocabsize = 5
memsize = 10
seqlen = 3
encdim = 13
data = np.random.randint(0, invocabsize, (memsize, seqlen))
gru = GRU(dim=invocabsize, innerdim=encdim)
payload = SeqEncoder(IdxToOneHot(vocsize=invocabsize), gru)
dynmemb = MemoryBlock(payload, outdim=encdim)
idxs = [0, 2, 5]
p = dynmemb.predict
memory_element = p(idxs, data)
self.assertEqual(memory_element.shape, (len(idxs), encdim))
gruparams = set([
getattr(gru, pname)
for pname in "u w b uhf whf bhf um wm bm".split()
])
allparams = set(p.outs[0].allparams)
self.assertEqual(gruparams.intersection(allparams), allparams)
statmemb = MemoryBlock(payload, data, outdim=encdim)
statpred = statmemb.predict(idxs)
self.assertTrue(np.allclose(statpred, memory_element))
def init(self):
#MEMORY: encodes how entity is written + custom entity embeddings
wencpg = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload, self.memdata, indim=self.outdim, outdim=self.encinnerdim+self.entembdim)
#ENCODER: uses the same language encoder as memory
#wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath)
self.enc = RecStack(wencpg, GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim))
#ATTENTION
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim + self.decinnerdim, innerdim=self.attdim)
attcon = WeightedSumAttCon()
#DECODER
#entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(
self.memaddr(
self.memblock,
indim=self.decinnerdim + self.encinnerdim,
memdim=self.memblock.outdim,
attdim=self.attdim),
Softmax())
self.dec = SeqDecoder(
[self.memblock, GRU(dim=self.entembdim + self.encinnerdim, innerdim=self.decinnerdim)],
outconcat=True, inconcat=False,
attention=Attention(attgen, attcon),
innerdim=self.decinnerdim + self.encinnerdim,
softmaxoutblock=self.softmaxoutblock
)
class Seq2Vec(Block):
def __init__(self, inpemb, enclayers, maskid=0, pool=None, **kw):
super(Seq2Vec, self).__init__(**kw)
self.maskid = maskid
self.inpemb = inpemb
if not issequence(enclayers):
enclayers = [enclayers]
self.pool = pool
self.enc = SeqEncoder(inpemb,
*enclayers).maskoptions(maskid, MaskMode.AUTO)
if self.pool is not None:
self.enc = self.enc.all_outputs.with_mask
def all_outputs(self):
self.enc = self.enc.all_outputs()
return self
def apply(self, x, mask=None, weights=None):
if self.pool is not None:
ret, mask = self.enc(x, mask=mask, weights=weights)
ret = self.pool(ret, mask)
else:
ret = self.enc(x, mask=mask, weights=weights)
return ret
def __init__(self, embedder, *layers, **kw):
super(SeqTrans, self).__init__(**kw)
self.enc = SeqEncoder(embedder, *layers)
self.enc.all_outputs().maskoption(MaskMode.NONE)
def __init__(self,
inpvocsize=None,
inpembdim=None,
inpemb=None,
inpencinnerdim=None,
bidir=False,
maskid=None,
dropout=False,
rnu=GRU,
inpencoder=None,
memvocsize=None,
memembdim=None,
memembmat=None,
memencinnerdim=None,
memencoder=None,
inp_att_dist=CosineDistance(),
mem_att_dist=CosineDistance(),
inp_attention=None,
mem_attention=None,
coredims=None,
corernu=GRU,
core=None,
explicit_interface=False,
scalaraggdim=None,
write_value_dim=None,
nsteps=100,
posvecdim=None,
mem_pos_repr=None,
inp_pos_repr=None,
inp_addr_extractor=None,
mem_addr_extractor=None,
write_addr_extractor=None,
write_addr_generator=None,
write_addr_dist=CosineDistance(),
write_value_generator=None,
write_value_extractor=None,
mem_erase_generator=None,
mem_change_generator=None,
memsampler=None,
memsamplemethod=None,
memsampletemp=0.3,
**kw):
# INPUT ENCODING
if inpencoder is None:
inpencoder = SeqEncoder.RNN(indim=inpvocsize,
inpembdim=inpembdim,
inpemb=inpemb,
innerdim=inpencinnerdim,
bidir=bidir,
maskid=maskid,
dropout_in=dropout,
dropout_h=dropout,
rnu=rnu).all_outputs()
lastinpdim = inpencinnerdim if not issequence(
inpencinnerdim) else inpencinnerdim[-1]
else:
lastinpdim = inpencoder.block.layers[-1].innerdim
# MEMORY ENCODING
if memembmat is None:
memembmat = param((memvocsize, memembdim),
name="memembmat").glorotuniform()
if memencoder is None:
memencoder = SeqEncoder.RNN(inpemb=False,
innerdim=memencinnerdim,
bidir=bidir,
dropout_in=dropout,
dropout_h=dropout,
rnu=rnu,
inpembdim=memembdim).all_outputs()
lastmemdim = memencinnerdim if not issequence(
memencinnerdim) else memencinnerdim[-1]
else:
lastmemdim = memencoder.block.layers[-1].innerdim
# POSITION VECTORS
if posvecdim is not None and inp_pos_repr is None:
inp_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout)
if posvecdim is not None and mem_pos_repr is None:
mem_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout)
xtra_dim = posvecdim if posvecdim is not None else 0
# CORE RNN - THE THINKER
if core is None:
corelayers, _ = MakeRNU.fromdims(
[lastinpdim + lastmemdim + xtra_dim * 2] + coredims,
rnu=corernu,
dropout_in=dropout,
dropout_h=dropout,
param_init_states=True)
core = RecStack(*corelayers)
lastcoredim = core.get_statespec()[-1][0][1][0]
# ATTENTIONS
if mem_attention is None:
mem_attention = Attention(mem_att_dist)
if inp_attention is None:
inp_attention = Attention(inp_att_dist)
if write_addr_generator is None:
write_addr_generator = AttGen(write_addr_dist)
# WRITE VALUE
if write_value_generator is None:
write_value_generator = WriteValGenerator(write_value_dim,
memvocsize,
dropout=dropout)
# MEMORY SAMPLER
if memsampler is not None:
assert (memsamplemethod is None)
if memsamplemethod is not None:
assert (memsampler is None)
memsampler = GumbelSoftmax(temperature=memsampletemp)
################ STATE INTERFACES #################
if not explicit_interface:
if inp_addr_extractor is None:
inp_addr_extractor = Forward(lastcoredim,
lastinpdim + xtra_dim,
dropout=dropout)
if mem_addr_extractor is None:
inp_addr_extractor = Forward(lastcoredim,
lastmemdim + xtra_dim,
dropout=dropout)
# WRITE INTERFACE
if write_addr_extractor is None:
write_addr_extractor = Forward(lastcoredim,
lastmemdim + xtra_dim,
dropout=dropout)
if write_value_extractor is None:
write_value_extractor = Forward(lastcoredim,
write_value_dim,
dropout=dropout)
# MEM UPDATE INTERFACE
if mem_erase_generator is None:
mem_erase_generator = StateToScalar(lastcoredim, scalaraggdim)
if mem_change_generator is None:
mem_change_generator = StateToScalar(lastcoredim, scalaraggdim)
else:
inp_addr_extractor, mem_addr_extractor, write_addr_extractor, \
write_value_extractor, mem_erase_generator, mem_change_generator = \
make_vector_slicers(0, lastinpdim + xtra_dim, lastmemdim + xtra_dim,
lastmemdim + xtra_dim, write_value_dim, 1, 1)
super(SimpleBulkNN,
self).__init__(inpencoder=inpencoder,
memembmat=memembmat,
memencoder=memencoder,
inp_attention=inp_attention,
mem_attention=mem_attention,
core=core,
memsampler=memsampler,
nsteps=nsteps,
inp_addr_extractor=inp_addr_extractor,
mem_addr_extractor=mem_addr_extractor,
write_addr_extractor=write_addr_extractor,
write_addr_generator=write_addr_generator,
mem_erase_generator=mem_erase_generator,
mem_change_generator=mem_change_generator,
write_value_generator=write_value_generator,
write_value_extractor=write_value_extractor,
inp_pos_repr=inp_pos_repr,
mem_pos_repr=mem_pos_repr,
**kw)
class SimpleSeq2MultiVec(Block):
def __init__(self,
indim=400,
inpembdim=50,
inpemb=None,
mode="concat",
innerdim=100,
numouts=1,
maskid=0,
bidir=False,
maskmode=MaskMode.NONE,
**kw):
super(SimpleSeq2MultiVec, self).__init__(**kw)
if inpemb is None:
if inpembdim is None:
inpemb = IdxToOneHot(indim)
inpembdim = indim
else:
inpemb = VectorEmbed(indim=indim, dim=inpembdim)
elif inpemb is False:
inpemb = None
else:
inpembdim = inpemb.outdim
if not issequence(innerdim):
innerdim = [innerdim]
innerdim[-1] += numouts
rnn, lastdim = self.makernu(inpembdim, innerdim, bidir=bidir)
self.outdim = lastdim * numouts if mode == "concat" else lastdim
self.maskid = maskid
self.inpemb = inpemb
self.numouts = numouts
self.mode = mode
self.bidir = bidir
if not issequence(rnn):
rnn = [rnn]
self.enc = SeqEncoder(inpemb, *rnn).maskoptions(maskid, maskmode)
self.enc.all_outputs()
@staticmethod
def makernu(inpembdim, innerdim, bidir=False):
return MakeRNU.make(inpembdim, innerdim, bidir=bidir)
def apply(self, x, mask=None, weights=None):
ret = self.enc(x, mask=mask,
weights=weights) # (batsize, seqlen, lastdim)
outs = []
# apply mask (SeqEncoder should attach mask to outvar if all_outputs()
mask = mask if mask is not None else ret.mask if hasattr(
ret, "mask") else None
if self.bidir:
mid = ret.shape[2] / 2
ret1 = ret[:, :, :mid]
ret2 = ret[:, :, mid:]
ret = ret1
for i in range(self.numouts):
selfweights = ret[:, :, i] # (batsize, seqlen)
if self.bidir:
selfweights += ret2[:, :, i]
selfweights = Softmax()(selfweights)
if mask is not None:
selfweights *= mask # apply mask
selfweights = selfweights / T.sum(selfweights, axis=1).dimshuffle(
0, "x") # renormalize
weightedstates = ret[:, :, self.numouts:] * selfweights.dimshuffle(
0, 1, "x")
if self.bidir:
weightedstates2 = ret2[:, :,
self.numouts:] * selfweights.dimshuffle(
0, 1, "x")
weightedstates = T.concatenate(
[weightedstates, weightedstates2], axis=2)
out = T.sum(weightedstates, axis=1) # (batsize, lastdim)
outs.append(out)
if self.mode == "concat":
ret = T.concatenate(outs, axis=1)
elif self.mode == "seq":
outs = [out.dimshuffle(0, "x", 1) for out in outs]
ret = T.concatenate(outs, axis=1)
return ret
