def __init__(self,
numchars=256,
charembdim=50,
charemb=None,
charinnerdim=100,
numwords=1000,
wordembdim=100,
wordemb=None,
wordinnerdim=200,
maskid=None,
bidir=False,
returnall=False,
**kw):
# char level inits
if charemb is None:
charemb = VectorEmbed(indim=numchars, dim=charembdim)
else:
charemb = charemb
charembdim = charemb.outdim
if not issequence(charinnerdim):
charinnerdim = [charinnerdim]
charlayers, lastchardim = MakeRNU.make(charembdim,
charinnerdim,
bidir=bidir)
charenc = SeqEncoder(charemb,
*charlayers).maskoptions(maskid, MaskMode.AUTO)
# word level inits
if wordemb is None:
wordemb = VectorEmbed(indim=numwords, dim=wordembdim)
elif wordemb is False:
wordemb = None
wordembdim = 0
else:
wordemb = wordemb
wordembdim = wordemb.outdim
if not issequence(wordinnerdim):
wordinnerdim = [wordinnerdim]
wordlayers, outdim = MakeRNU.make(wordembdim + lastchardim,
wordinnerdim,
bidir=bidir)
wordenc = SeqEncoder(None, *wordlayers).maskoptions(MaskMode.NONE)
if returnall:
wordenc.all_outputs()
self.outdim = outdim
super(WordCharSentEnc, self).__init__(l1enc=charenc,
l2emb=wordemb,
l2enc=wordenc,
maskid=maskid)
def __init__(self,
indim=400,
inpembdim=50,
inpemb=None,
innerdim=100,
maskid=None,
bidir=False,
**kw):
if inpemb is None:
if inpembdim is None:
inpemb = IdxToOneHot(indim)
inpembdim = indim
else:
inpemb = VectorEmbed(indim=indim, dim=inpembdim)
else:
inpembdim = inpemb.outdim
lastdim = inpembdim
if not issequence(innerdim): # single encoder
innerdim = [innerdim]
rnns = []
for innerdimi in innerdim:
if not issequence(innerdimi): # one layer in encoder
innerdimi = [innerdimi]
rnn, lastdim = MakeRNU.make(lastdim, innerdimi, bidir=bidir)
rnns.append(rnn)
self.outdim = lastdim
super(SimpleSeqStar2Vec, self).__init__(inpemb,
*rnns,
maskid=maskid,
**kw)
def makernu(inpembdim,
innerdim,
bidir=False,
zoneout=False,
dropout_h=False,
dropout_in=False):
return MakeRNU.make(inpembdim,
innerdim,
bidir=bidir,
zoneout=zoneout,
dropout_h=dropout_h,
dropout_in=dropout_in)
def __init__(self,
indim=400,
outdim=50,
inpembdim=50,
outembdim=50,
innerdim=100,
**kw):
self.inpemb = VectorEmbed(indim=indim, dim=inpembdim)
self.outemb = VectorEmbed(indim=outdim, dim=outembdim)
self.rnn = []
if not issequence(innerdim):
innerdim = [innerdim]
innerdim = [inpembdim + outembdim] + innerdim
self.rnn = MakeRNU.fromdims(innerdim)[0]
super(SimpleSeqTransDec, self).__init__(self.inpemb,
self.outemb,
*self.rnn,
smodim=innerdim[-1],
outdim=outdim,
**kw)
def __init__(self,
indim=400,
embdim=50,
inpemb=None,
innerdim=100,
outdim=50,
rnu=GRU,
**kw):
if inpemb is None:
emb = VectorEmbed(indim=indim, dim=embdim)
else:
emb = inpemb
embdim = emb.outdim
if not issequence(innerdim):
innerdim = [innerdim]
innerdim = [embdim] + innerdim
rnn, _ = MakeRNU.fromdims(innerdim, rnu=rnu)
smo = Lin(indim=innerdim[-1], dim=outdim)
super(SimpleSeqTrans, self).__init__(emb, *(rnn + [smo, Softmax()]),
**kw)
def __init__(self,
indim=400,
embdim=50,
inpemb=None,
innerdim=100,
outdim=50,
rnu=GRU,
**kw):
if inpemb is None:
self.emb = VectorEmbed(indim=indim, dim=embdim)
else:
self.emb = inpemb
embdim = self.emb.outdim
if not issequence(innerdim):
innerdim = [innerdim]
innerdim = [embdim] + innerdim
self.rnn = MakeRNU.fromdims(innerdim, rnu=rnu)[0]
super(SimpleSeqTransducer, self).__init__(self.emb,
*self.rnn,
smodim=innerdim[-1],
outdim=outdim,
**kw)
def __init__(self,
inpvocsize=None,
inpembdim=None,
inpemb=None,
inpencinnerdim=None,
bidir=False,
maskid=None,
dropout=False,
rnu=GRU,
inpencoder=None,
memvocsize=None,
memembdim=None,
memembmat=None,
memencinnerdim=None,
memencoder=None,
inp_att_dist=CosineDistance(),
mem_att_dist=CosineDistance(),
inp_attention=None,
mem_attention=None,
coredims=None,
corernu=GRU,
core=None,
explicit_interface=False,
scalaraggdim=None,
write_value_dim=None,
nsteps=100,
posvecdim=None,
mem_pos_repr=None,
inp_pos_repr=None,
inp_addr_extractor=None,
mem_addr_extractor=None,
write_addr_extractor=None,
write_addr_generator=None,
write_addr_dist=CosineDistance(),
write_value_generator=None,
write_value_extractor=None,
mem_erase_generator=None,
mem_change_generator=None,
memsampler=None,
memsamplemethod=None,
memsampletemp=0.3,
**kw):
# INPUT ENCODING
if inpencoder is None:
inpencoder = SeqEncoder.RNN(indim=inpvocsize,
inpembdim=inpembdim,
inpemb=inpemb,
innerdim=inpencinnerdim,
bidir=bidir,
maskid=maskid,
dropout_in=dropout,
dropout_h=dropout,
rnu=rnu).all_outputs()
lastinpdim = inpencinnerdim if not issequence(
inpencinnerdim) else inpencinnerdim[-1]
else:
lastinpdim = inpencoder.block.layers[-1].innerdim
# MEMORY ENCODING
if memembmat is None:
memembmat = param((memvocsize, memembdim),
name="memembmat").glorotuniform()
if memencoder is None:
memencoder = SeqEncoder.RNN(inpemb=False,
innerdim=memencinnerdim,
bidir=bidir,
dropout_in=dropout,
dropout_h=dropout,
rnu=rnu,
inpembdim=memembdim).all_outputs()
lastmemdim = memencinnerdim if not issequence(
memencinnerdim) else memencinnerdim[-1]
else:
lastmemdim = memencoder.block.layers[-1].innerdim
# POSITION VECTORS
if posvecdim is not None and inp_pos_repr is None:
inp_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout)
if posvecdim is not None and mem_pos_repr is None:
mem_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout)
xtra_dim = posvecdim if posvecdim is not None else 0
# CORE RNN - THE THINKER
if core is None:
corelayers, _ = MakeRNU.fromdims(
[lastinpdim + lastmemdim + xtra_dim * 2] + coredims,
rnu=corernu,
dropout_in=dropout,
dropout_h=dropout,
param_init_states=True)
core = RecStack(*corelayers)
lastcoredim = core.get_statespec()[-1][0][1][0]
# ATTENTIONS
if mem_attention is None:
mem_attention = Attention(mem_att_dist)
if inp_attention is None:
inp_attention = Attention(inp_att_dist)
if write_addr_generator is None:
write_addr_generator = AttGen(write_addr_dist)
# WRITE VALUE
if write_value_generator is None:
write_value_generator = WriteValGenerator(write_value_dim,
memvocsize,
dropout=dropout)
# MEMORY SAMPLER
if memsampler is not None:
assert (memsamplemethod is None)
if memsamplemethod is not None:
assert (memsampler is None)
memsampler = GumbelSoftmax(temperature=memsampletemp)
################ STATE INTERFACES #################
if not explicit_interface:
if inp_addr_extractor is None:
inp_addr_extractor = Forward(lastcoredim,
lastinpdim + xtra_dim,
dropout=dropout)
if mem_addr_extractor is None:
inp_addr_extractor = Forward(lastcoredim,
lastmemdim + xtra_dim,
dropout=dropout)
# WRITE INTERFACE
if write_addr_extractor is None:
write_addr_extractor = Forward(lastcoredim,
lastmemdim + xtra_dim,
dropout=dropout)
if write_value_extractor is None:
write_value_extractor = Forward(lastcoredim,
write_value_dim,
dropout=dropout)
# MEM UPDATE INTERFACE
if mem_erase_generator is None:
mem_erase_generator = StateToScalar(lastcoredim, scalaraggdim)
if mem_change_generator is None:
mem_change_generator = StateToScalar(lastcoredim, scalaraggdim)
else:
inp_addr_extractor, mem_addr_extractor, write_addr_extractor, \
write_value_extractor, mem_erase_generator, mem_change_generator = \
make_vector_slicers(0, lastinpdim + xtra_dim, lastmemdim + xtra_dim,
lastmemdim + xtra_dim, write_value_dim, 1, 1)
super(SimpleBulkNN,
self).__init__(inpencoder=inpencoder,
memembmat=memembmat,
memencoder=memencoder,
inp_attention=inp_attention,
mem_attention=mem_attention,
core=core,
memsampler=memsampler,
nsteps=nsteps,
inp_addr_extractor=inp_addr_extractor,
mem_addr_extractor=mem_addr_extractor,
write_addr_extractor=write_addr_extractor,
write_addr_generator=write_addr_generator,
mem_erase_generator=mem_erase_generator,
mem_change_generator=mem_change_generator,
write_value_generator=write_value_generator,
write_value_extractor=write_value_extractor,
inp_pos_repr=inp_pos_repr,
mem_pos_repr=mem_pos_repr,
**kw)
def makernu(inpembdim, innerdim, bidir=False):
return MakeRNU.make(inpembdim, innerdim, bidir=bidir)
def getrnnfrominnerdim(self, innerdim, rnu=GRU):
assert (len(innerdim) >= 2)
initdim = innerdim[0]
otherdim = innerdim[1:]
return MakeRNU.make(initdim, otherdim, rnu=rnu)[0]
def makernu(inpembdim, innerdim, bidir=False):
return MakeRNU.make(inpembdim, innerdim, bidir=bidir)
def getrnnfrominnerdim(self, innerdim, rnu=GRU):
assert(len(innerdim) >= 2)
initdim = innerdim[0]
otherdim = innerdim[1:]
return MakeRNU.make(initdim, otherdim, rnu=rnu)[0]
