class SeqTransDec(Block):
def __init__(self, *layers, **kw):
""" first two layers must be embedding layers. Final softmax is added automatically"""
assert ("smodim" in kw and "outdim" in kw)
smodim = kw["smodim"]
outdim = kw["outdim"]
del kw["smodim"]
del kw["outdim"]
super(SeqTransDec, self).__init__(**kw)
self.inpemb = layers[0]
self.outemb = layers[1]
self.block = RecStack(*(layers[2:] +
(Lin(indim=smodim, dim=outdim), Softmax())))
def apply(self, inpseq, outseq, maskseq=None):
# embed with the two embedding layers
emb = self._get_emb(inpseq, outseq)
res = self.block(emb)
ret = SeqTransducer.applymask(res, maskseq=maskseq)
return ret
def _get_emb(self, inpseq, outseq):
iemb = self.inpemb(inpseq) # (batsize, seqlen, inpembdim)
oemb = self.outemb(outseq) # (batsize, seqlen, outembdim)
emb = T.concatenate([iemb, oemb], axis=iemb.ndim -
1) # (batsize, seqlen, inpembdim+outembdim)
return emb
def rec(self, inpa, inpb, *states):
emb = self._get_emb(inpa, inpb)
return self.block.rec(emb, *states)
def get_init_info(self, initstates):
return self.block.get_init_info(initstates)
class SeqTransDec(Block):
def __init__(self, *layers, **kw):
""" first two layers must be embedding layers. Final softmax is added automatically"""
assert("smodim" in kw and "outdim" in kw)
smodim = kw["smodim"]
outdim = kw["outdim"]
del kw["smodim"]; del kw["outdim"]
super(SeqTransDec, self).__init__(**kw)
self.inpemb = layers[0]
self.outemb = layers[1]
self.block = RecStack(*(layers[2:] + (Lin(indim=smodim, dim=outdim), Softmax())))
def apply(self, inpseq, outseq, maskseq=None):
# embed with the two embedding layers
emb = self._get_emb(inpseq, outseq)
res = self.block(emb)
ret = SeqTransducer.applymask(res, maskseq=maskseq)
return ret
def _get_emb(self, inpseq, outseq):
iemb = self.inpemb(inpseq) # (batsize, seqlen, inpembdim)
oemb = self.outemb(outseq) # (batsize, seqlen, outembdim)
emb = T.concatenate([iemb, oemb], axis=iemb.ndim-1) # (batsize, seqlen, inpembdim+outembdim)
return emb
def rec(self, inpa, inpb, *states):
emb = self._get_emb(inpa, inpb)
return self.block.rec(emb, *states)
def get_init_info(self, initstates):
return self.block.get_init_info(initstates)
def __init__(self, *layers, **kw):
""" first two layers must be embedding layers. Final softmax is added automatically"""
assert ("smodim" in kw and "outdim" in kw)
smodim = kw["smodim"]
outdim = kw["outdim"]
del kw["smodim"]
del kw["outdim"]
super(SeqTransDec, self).__init__(**kw)
self.inpemb = layers[0]
self.outemb = layers[1]
self.block = RecStack(*(layers[2:] +
(Lin(indim=smodim, dim=outdim), Softmax())))
def __init__(self,
wordembdim=50,
entembdim=200,
innerdim=200,
attdim=100,
outdim=1e4,
numwords=4e5,
**kw):
super(FBSeqSimpEncDecAtt, self).__init__(**kw)
self.indim = wordembdim
self.outdim = outdim
self.wordembdim = wordembdim
self.encinnerdim = innerdim
self.decinnerdim = innerdim
self.entembdim = entembdim
self.wordencoder = WordEmbed(indim=numwords,
outdim=self.wordembdim,
trainfrac=1.0)
self.rnn = RecStack(
self.wordencoder,
GRU(dim=self.wordembdim, innerdim=self.encinnerdim))
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
attdim=attdim)
attcon = WeightedSumAttCon()
self.dec = SeqDecoder([
VectorEmbed(indim=self.outdim, dim=self.entembdim),
GRU(dim=self.entembdim, innerdim=self.decinnerdim)
],
attention=Attention(attgen, attcon),
outconcat=True,
inconcat=False,
innerdim=self.encinnerdim + self.decinnerdim)
def stack(*layers, **kw):
rec = False
for layer in layers:
if isinstance(layer, RNUBase):
rec = True
break
if rec is True:
return RecStack(*layers, **kw)
else:
return BlockStack(*layers, **kw)
def __init__(self, *layers, **kw):
""" first two layers must be embedding layers. Final softmax is added automatically"""
assert("smodim" in kw and "outdim" in kw)
smodim = kw["smodim"]
outdim = kw["outdim"]
del kw["smodim"]; del kw["outdim"]
super(SeqTransDec, self).__init__(**kw)
self.inpemb = layers[0]
self.outemb = layers[1]
self.block = RecStack(*(layers[2:] + (Lin(indim=smodim, dim=outdim), Softmax())))
def __init__(self, embedder, *layers, **kw):
""" layers must have an embedding layers first, final softmax layer is added automatically"""
assert ("smodim" in kw and "outdim" in kw)
self.embedder = embedder
smodim = kw["smodim"]
outdim = kw["outdim"]
del kw["smodim"]
del kw["outdim"]
super(SeqTransducer, self).__init__(**kw)
self.block = RecStack(*(layers +
(Lin(indim=smodim, dim=outdim), Softmax())))
def test_recappl(self):
batsize = 100
self.dims = [50, 20, 30, 40]
recstack = RecStack(*[
GRU(dim=self.dims[i], innerdim=self.dims[i + 1])
for i in range(len(self.dims) - 1)
])
mu = RecPredictor(recstack).init(batsize)
for i in range(3):
inpval = np.random.random((batsize, 50)).astype("float32")
outpvals = mu.feed(inpval)
self.assertEqual(outpvals.shape, (batsize, self.dims[-1]))
def init(self):
#MEMORY: encodes how entity is written + custom entity embeddings
wencpg = WordEncoderPlusGlove(numchars=self.numchars,
numwords=self.numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload,
self.memdata,
indim=self.outdim,
outdim=self.encinnerdim + self.entembdim)
#ENCODER: uses the same language encoder as memory
#wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath)
self.enc = RecStack(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
#ATTENTION
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
innerdim=self.attdim)
attcon = WeightedSumAttCon()
#DECODER
#entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(
self.memaddr(self.memblock,
indim=self.decinnerdim + self.encinnerdim,
memdim=self.memblock.outdim,
attdim=self.attdim), Softmax())
self.dec = SeqDecoder([
self.memblock,
GRU(dim=self.entembdim + self.encinnerdim,
innerdim=self.decinnerdim)
],
outconcat=True,
inconcat=False,
attention=Attention(attgen, attcon),
innerdim=self.decinnerdim + self.encinnerdim,
softmaxoutblock=self.softmaxoutblock)
def __init__(self,
wordembdim=50,
wordencdim=50,
entembdim=200,
innerdim=200,
attdim=100,
outdim=1e4,
numwords=4e5,
numchars=128,
glovepath=None,
**kw):
super(FBSeqCompEncDecAtt, self).__init__(**kw)
self.indim = wordembdim + wordencdim
self.outdim = outdim
self.wordembdim = wordembdim
self.wordencdim = wordencdim
self.encinnerdim = innerdim
self.entembdim = entembdim
self.decinnerdim = innerdim
self.wordencoder = WordEncoderPlusGlove(numchars=numchars,
numwords=numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=glovepath)
self.rnn = RecStack(
self.wordencoder,
GRU(dim=wordembdim + wordencdim, innerdim=self.encinnerdim))
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
innerdim=attdim)
attcon = WeightedSumAttCon()
self.dec = SeqDecoder([
VectorEmbed(indim=self.outdim, dim=self.entembdim),
GRU(dim=self.entembdim, innerdim=self.decinnerdim)
],
attention=Attention(attgen, attcon),
outconcat=True,
inconcat=False,
innerdim=self.encinnerdim + self.decinnerdim)
def __init__(self,
inpvocsize=None,
inpembdim=None,
inpemb=None,
inpencinnerdim=None,
bidir=False,
maskid=None,
dropout=False,
rnu=GRU,
inpencoder=None,
memvocsize=None,
memembdim=None,
memembmat=None,
memencinnerdim=None,
memencoder=None,
inp_att_dist=CosineDistance(),
mem_att_dist=CosineDistance(),
inp_attention=None,
mem_attention=None,
coredims=None,
corernu=GRU,
core=None,
explicit_interface=False,
scalaraggdim=None,
write_value_dim=None,
nsteps=100,
posvecdim=None,
mem_pos_repr=None,
inp_pos_repr=None,
inp_addr_extractor=None,
mem_addr_extractor=None,
write_addr_extractor=None,
write_addr_generator=None,
write_addr_dist=CosineDistance(),
write_value_generator=None,
write_value_extractor=None,
mem_erase_generator=None,
mem_change_generator=None,
memsampler=None,
memsamplemethod=None,
memsampletemp=0.3,
**kw):
# INPUT ENCODING
if inpencoder is None:
inpencoder = SeqEncoder.RNN(indim=inpvocsize,
inpembdim=inpembdim,
inpemb=inpemb,
innerdim=inpencinnerdim,
bidir=bidir,
maskid=maskid,
dropout_in=dropout,
dropout_h=dropout,
rnu=rnu).all_outputs()
lastinpdim = inpencinnerdim if not issequence(
inpencinnerdim) else inpencinnerdim[-1]
else:
lastinpdim = inpencoder.block.layers[-1].innerdim
# MEMORY ENCODING
if memembmat is None:
memembmat = param((memvocsize, memembdim),
name="memembmat").glorotuniform()
if memencoder is None:
memencoder = SeqEncoder.RNN(inpemb=False,
innerdim=memencinnerdim,
bidir=bidir,
dropout_in=dropout,
dropout_h=dropout,
rnu=rnu,
inpembdim=memembdim).all_outputs()
lastmemdim = memencinnerdim if not issequence(
memencinnerdim) else memencinnerdim[-1]
else:
lastmemdim = memencoder.block.layers[-1].innerdim
# POSITION VECTORS
if posvecdim is not None and inp_pos_repr is None:
inp_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout)
if posvecdim is not None and mem_pos_repr is None:
mem_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout)
xtra_dim = posvecdim if posvecdim is not None else 0
# CORE RNN - THE THINKER
if core is None:
corelayers, _ = MakeRNU.fromdims(
[lastinpdim + lastmemdim + xtra_dim * 2] + coredims,
rnu=corernu,
dropout_in=dropout,
dropout_h=dropout,
param_init_states=True)
core = RecStack(*corelayers)
lastcoredim = core.get_statespec()[-1][0][1][0]
# ATTENTIONS
if mem_attention is None:
mem_attention = Attention(mem_att_dist)
if inp_attention is None:
inp_attention = Attention(inp_att_dist)
if write_addr_generator is None:
write_addr_generator = AttGen(write_addr_dist)
# WRITE VALUE
if write_value_generator is None:
write_value_generator = WriteValGenerator(write_value_dim,
memvocsize,
dropout=dropout)
# MEMORY SAMPLER
if memsampler is not None:
assert (memsamplemethod is None)
if memsamplemethod is not None:
assert (memsampler is None)
memsampler = GumbelSoftmax(temperature=memsampletemp)
################ STATE INTERFACES #################
if not explicit_interface:
if inp_addr_extractor is None:
inp_addr_extractor = Forward(lastcoredim,
lastinpdim + xtra_dim,
dropout=dropout)
if mem_addr_extractor is None:
inp_addr_extractor = Forward(lastcoredim,
lastmemdim + xtra_dim,
dropout=dropout)
# WRITE INTERFACE
if write_addr_extractor is None:
write_addr_extractor = Forward(lastcoredim,
lastmemdim + xtra_dim,
dropout=dropout)
if write_value_extractor is None:
write_value_extractor = Forward(lastcoredim,
write_value_dim,
dropout=dropout)
# MEM UPDATE INTERFACE
if mem_erase_generator is None:
mem_erase_generator = StateToScalar(lastcoredim, scalaraggdim)
if mem_change_generator is None:
mem_change_generator = StateToScalar(lastcoredim, scalaraggdim)
else:
inp_addr_extractor, mem_addr_extractor, write_addr_extractor, \
write_value_extractor, mem_erase_generator, mem_change_generator = \
make_vector_slicers(0, lastinpdim + xtra_dim, lastmemdim + xtra_dim,
lastmemdim + xtra_dim, write_value_dim, 1, 1)
super(SimpleBulkNN,
self).__init__(inpencoder=inpencoder,
memembmat=memembmat,
memencoder=memencoder,
inp_attention=inp_attention,
mem_attention=mem_attention,
core=core,
memsampler=memsampler,
nsteps=nsteps,
inp_addr_extractor=inp_addr_extractor,
mem_addr_extractor=mem_addr_extractor,
write_addr_extractor=write_addr_extractor,
write_addr_generator=write_addr_generator,
mem_erase_generator=mem_erase_generator,
mem_change_generator=mem_change_generator,
write_value_generator=write_value_generator,
write_value_extractor=write_value_extractor,
inp_pos_repr=inp_pos_repr,
mem_pos_repr=mem_pos_repr,
**kw)
