def setUp(self):
criteriondim = 20
datadim = 20
innerdim = 30
batsize = 33
seqlen = 11
self.attgenshape = (batsize, seqlen)
self.attconshape = (batsize, datadim)
self.attgen = self.getattgenc(critdim=criteriondim, datadim=datadim, attdim=innerdim)
self.attgenparams = self.getattgenparams()
self.attcon = WeightedSumAttCon()
self.att = Attention(self.attgen, self.attcon)
self.criterion_val = np.random.random((batsize, criteriondim)).astype("float32")
self.data_val = np.random.random((batsize, seqlen, datadim)).astype("float32")
def __init__(self,
enclayers,
declayers,
attgen,
attcon,
decinnerdim,
statetrans=None,
vecout=False,
inconcat=True,
outconcat=False,
**kw):
enc = SeqEncoder(*enclayers)\
.with_outputs()\
.with_mask()\
.maskoptions(-1, MaskMode.AUTO, MaskSetMode.ZERO)
smo = False if vecout else None
dec = SeqDecoder(declayers,
attention=Attention(attgen, attcon),
innerdim=decinnerdim,
inconcat=inconcat,
softmaxoutblock=smo,
outconcat=outconcat)
super(SeqEncDecAtt, self).__init__(enc,
dec,
statetrans=statetrans,
**kw)
def __init__(self,
wordembdim=50,
entembdim=200,
innerdim=200,
attdim=100,
outdim=1e4,
numwords=4e5,
**kw):
super(FBSeqSimpEncDecAtt, self).__init__(**kw)
self.indim = wordembdim
self.outdim = outdim
self.wordembdim = wordembdim
self.encinnerdim = innerdim
self.decinnerdim = innerdim
self.entembdim = entembdim
self.wordencoder = WordEmbed(indim=numwords,
outdim=self.wordembdim,
trainfrac=1.0)
self.rnn = RecStack(
self.wordencoder,
GRU(dim=self.wordembdim, innerdim=self.encinnerdim))
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
attdim=attdim)
attcon = WeightedSumAttCon()
self.dec = SeqDecoder([
VectorEmbed(indim=self.outdim, dim=self.entembdim),
GRU(dim=self.entembdim, innerdim=self.decinnerdim)
],
attention=Attention(attgen, attcon),
outconcat=True,
inconcat=False,
innerdim=self.encinnerdim + self.decinnerdim)
def setUp(self):
vocsize = 10
innerdim = 50
encdim = 30
seqlen = 5
batsize = 77
self.att = Attention(
LinearSumAttentionGenerator(indim=innerdim + encdim),
WeightedSumAttCon())
self.decwatt = SeqDecoder([
IdxToOneHot(vocsize),
GRU(dim=vocsize + encdim, innerdim=innerdim)
],
inconcat=True,
attention=self.att,
innerdim=innerdim)
self.decwoatt = SeqDecoder([
IdxToOneHot(vocsize),
GRU(dim=vocsize + encdim, innerdim=innerdim)
],
inconcat=True,
innerdim=innerdim)
self.attdata = np.random.random(
(batsize, seqlen, encdim)).astype("float32")
self.data = np.random.random((batsize, encdim)).astype("float32")
self.seqdata = np.random.randint(0, vocsize, (batsize, seqlen))
self.predshape = (batsize, seqlen, vocsize)
def __init__(self, vocsize=25, outvocsize=25, encdim=200, innerdim=200, attdim=50, **kw):
super(RewAttSumDecoder, self).__init__(**kw)
self.rnn = SeqEncoder(IdxToOneHot(vocsize), GRU(dim=vocsize, innerdim=encdim)).all_outputs
attgen = LinearGateAttentionGenerator(indim=innerdim+encdim, innerdim=attdim)
attcon = WeightedSumAttCon()
self.dec = SeqDecoder([IdxToOneHot(outvocsize), GRU(dim=outvocsize+encdim, innerdim=innerdim)],
inconcat=True,
attention=Attention(attgen, attcon),
innerdim=innerdim)
class DummyAttentionGeneratorConsumerTest(TestCase):
def setUp(self):
criteriondim = 30
datadim = 20
innerdim = 25
batsize = 33
seqlen = 11
self.attgenshape = (batsize, seqlen)
self.attconshape = (batsize, datadim)
self.attgenc = self.getattgenc()
self.attgen = self.attgenc(indim=criteriondim + datadim,
innerdim=innerdim)
self.attgenparams = self.getattgenparams()
self.attcon = WeightedSumAttCon()
self.att = Attention(self.attgen, self.attcon)
self.criterion_val = np.random.random(
(batsize, criteriondim)).astype("float32")
self.data_val = np.random.random(
(batsize, seqlen, datadim)).astype("float32")
def getattgenc(self):
return LinearSumAttentionGenerator
def getattgenparams(self):
return {self.attgen.W}
def test_generator_shape(self):
pred = self.attgen.predict(self.criterion_val, self.data_val)
self.assertEqual(pred.shape, self.attgenshape)
def test_generator_param_prop(self):
self.attgen.predict(self.criterion_val, self.data_val)
allparams = self.attgen.output.allparams
self.assertSetEqual(allparams, self.attgenparams)
def test_consumer_shape(self):
pred = self.att.predict(self.criterion_val, self.data_val)
self.assertEqual(pred.shape, self.attconshape)
def test_consumer_param_prop(self):
self.att.predict(self.criterion_val, self.data_val)
allparams = self.att.output.allparams
self.assertSetEqual(allparams, self.attgenparams)
def __init__(self, vocsize=25, outvocsize=20, encdim=200, innerdim=200, attdim=50, **kw):
super(FwdAttRNNEncDecoder, self).__init__(**kw)
self.emb = IdxToOneHot(vocsize)
attgen = LinearGateAttentionGenerator(indim=innerdim+vocsize, innerdim=attdim)
attcon = SeqEncoder(None,
GRU(dim=vocsize, innerdim=encdim))
self.dec = SeqDecoder([IdxToOneHot(outvocsize), GRU(dim=outvocsize, innerdim=innerdim)],
outconcat=True, inconcat=False,
attention=Attention(attgen, attcon),
innerdim=innerdim+encdim)
class DummyAttentionGeneratorConsumerTest(TestCase):
def setUp(self):
criteriondim = 30
datadim = 20
innerdim = 25
batsize = 33
seqlen = 11
self.attgenshape = (batsize, seqlen)
self.attconshape = (batsize, datadim)
self.attgenc = self.getattgenc()
self.attgen = self.attgenc(indim=criteriondim + datadim, innerdim=innerdim)
self.attgenparams = self.getattgenparams()
self.attcon = WeightedSumAttCon()
self.att = Attention(self.attgen, self.attcon)
self.criterion_val = np.random.random((batsize, criteriondim)).astype("float32")
self.data_val = np.random.random((batsize, seqlen, datadim)).astype("float32")
def getattgenc(self):
return LinearSumAttentionGenerator
def getattgenparams(self):
return {self.attgen.W}
def test_generator_shape(self):
pred = self.attgen.predict(self.criterion_val, self.data_val)
self.assertEqual(pred.shape, self.attgenshape)
def test_generator_param_prop(self):
self.attgen.predict(self.criterion_val, self.data_val)
allparams = self.attgen.output.allparams
self.assertSetEqual(allparams, self.attgenparams)
def test_consumer_shape(self):
pred = self.att.predict(self.criterion_val, self.data_val)
self.assertEqual(pred.shape, self.attconshape)
def test_consumer_param_prop(self):
self.att.predict(self.criterion_val, self.data_val)
allparams = self.att.output.allparams
self.assertSetEqual(allparams, self.attgenparams)
class DummyAttentionGeneratorConsumerTest(TestCase):
def setUp(self):
criteriondim = 20
datadim = 20
innerdim = 30
batsize = 33
seqlen = 11
self.attgenshape = (batsize, seqlen)
self.attconshape = (batsize, datadim)
self.attgen = self.getattgenc(critdim=criteriondim, datadim=datadim, attdim=innerdim)
self.attgenparams = self.getattgenparams()
self.attcon = WeightedSumAttCon()
self.att = Attention(self.attgen, self.attcon)
self.criterion_val = np.random.random((batsize, criteriondim)).astype("float32")
self.data_val = np.random.random((batsize, seqlen, datadim)).astype("float32")
def getattgenc(self, critdim=None, datadim=None, attdim=None):
return AttGen(DotDistance())
def getattgenparams(self):
return set()
def test_generator_shape(self):
pred = self.attgen.predict(self.criterion_val, self.data_val)
self.assertEqual(pred.shape, self.attgenshape)
def test_generator_param_prop(self):
_, outps = self.attgen.autobuild(self.criterion_val, self.data_val)
allparams = outps[0].allparams
self.assertSetEqual(allparams, self.attgenparams)
def test_consumer_shape(self):
pred = self.att.predict(self.criterion_val, self.data_val)
self.assertEqual(pred.shape, self.attconshape)
def test_consumer_param_prop(self):
_, outps = self.att.autobuild(self.criterion_val, self.data_val)
allparams = outps[0].allparams
self.assertSetEqual(allparams, self.attgenparams)
def init(self):
#MEMORY: encodes how entity is written + custom entity embeddings
wencpg = WordEncoderPlusGlove(numchars=self.numchars,
numwords=self.numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload,
self.memdata,
indim=self.outdim,
outdim=self.encinnerdim + self.entembdim)
#ENCODER: uses the same language encoder as memory
#wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath)
self.enc = RecStack(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
#ATTENTION
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
innerdim=self.attdim)
attcon = WeightedSumAttCon()
#DECODER
#entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(
self.memaddr(self.memblock,
indim=self.decinnerdim + self.encinnerdim,
memdim=self.memblock.outdim,
attdim=self.attdim), Softmax())
self.dec = SeqDecoder([
self.memblock,
GRU(dim=self.entembdim + self.encinnerdim,
innerdim=self.decinnerdim)
],
outconcat=True,
inconcat=False,
attention=Attention(attgen, attcon),
innerdim=self.decinnerdim + self.encinnerdim,
softmaxoutblock=self.softmaxoutblock)
def setUp(self):
criteriondim = 30
datadim = 20
innerdim = 25
batsize = 33
seqlen = 11
self.attgenshape = (batsize, seqlen)
self.attconshape = (batsize, datadim)
self.attgenc = self.getattgenc()
self.attgen = self.attgenc(indim=criteriondim + datadim, innerdim=innerdim)
self.attgenparams = self.getattgenparams()
self.attcon = WeightedSumAttCon()
self.att = Attention(self.attgen, self.attcon)
self.criterion_val = np.random.random((batsize, criteriondim)).astype("float32")
self.data_val = np.random.random((batsize, seqlen, datadim)).astype("float32")
def __init__(self,
wordembdim=50,
wordencdim=50,
entembdim=200,
innerdim=200,
attdim=100,
outdim=1e4,
numwords=4e5,
numchars=128,
glovepath=None,
**kw):
super(FBSeqCompEncDecAtt, self).__init__(**kw)
self.indim = wordembdim + wordencdim
self.outdim = outdim
self.wordembdim = wordembdim
self.wordencdim = wordencdim
self.encinnerdim = innerdim
self.entembdim = entembdim
self.decinnerdim = innerdim
self.wordencoder = WordEncoderPlusGlove(numchars=numchars,
numwords=numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=glovepath)
self.rnn = RecStack(
self.wordencoder,
GRU(dim=wordembdim + wordencdim, innerdim=self.encinnerdim))
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
innerdim=attdim)
attcon = WeightedSumAttCon()
self.dec = SeqDecoder([
VectorEmbed(indim=self.outdim, dim=self.entembdim),
GRU(dim=self.entembdim, innerdim=self.decinnerdim)
],
attention=Attention(attgen, attcon),
outconcat=True,
inconcat=False,
innerdim=self.encinnerdim + self.decinnerdim)
def __init__(self,
inpvocsize=None,
inpembdim=None,
inpemb=None,
inpencinnerdim=None,
bidir=False,
maskid=None,
dropout=False,
rnu=GRU,
inpencoder=None,
memvocsize=None,
memembdim=None,
memembmat=None,
memencinnerdim=None,
memencoder=None,
inp_att_dist=CosineDistance(),
mem_att_dist=CosineDistance(),
inp_attention=None,
mem_attention=None,
coredims=None,
corernu=GRU,
core=None,
explicit_interface=False,
scalaraggdim=None,
write_value_dim=None,
nsteps=100,
posvecdim=None,
mem_pos_repr=None,
inp_pos_repr=None,
inp_addr_extractor=None,
mem_addr_extractor=None,
write_addr_extractor=None,
write_addr_generator=None,
write_addr_dist=CosineDistance(),
write_value_generator=None,
write_value_extractor=None,
mem_erase_generator=None,
mem_change_generator=None,
memsampler=None,
memsamplemethod=None,
memsampletemp=0.3,
**kw):
# INPUT ENCODING
if inpencoder is None:
inpencoder = SeqEncoder.RNN(indim=inpvocsize,
inpembdim=inpembdim,
inpemb=inpemb,
innerdim=inpencinnerdim,
bidir=bidir,
maskid=maskid,
dropout_in=dropout,
dropout_h=dropout,
rnu=rnu).all_outputs()
lastinpdim = inpencinnerdim if not issequence(
inpencinnerdim) else inpencinnerdim[-1]
else:
lastinpdim = inpencoder.block.layers[-1].innerdim
# MEMORY ENCODING
if memembmat is None:
memembmat = param((memvocsize, memembdim),
name="memembmat").glorotuniform()
if memencoder is None:
memencoder = SeqEncoder.RNN(inpemb=False,
innerdim=memencinnerdim,
bidir=bidir,
dropout_in=dropout,
dropout_h=dropout,
rnu=rnu,
inpembdim=memembdim).all_outputs()
lastmemdim = memencinnerdim if not issequence(
memencinnerdim) else memencinnerdim[-1]
else:
lastmemdim = memencoder.block.layers[-1].innerdim
# POSITION VECTORS
if posvecdim is not None and inp_pos_repr is None:
inp_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout)
if posvecdim is not None and mem_pos_repr is None:
mem_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout)
xtra_dim = posvecdim if posvecdim is not None else 0
# CORE RNN - THE THINKER
if core is None:
corelayers, _ = MakeRNU.fromdims(
[lastinpdim + lastmemdim + xtra_dim * 2] + coredims,
rnu=corernu,
dropout_in=dropout,
dropout_h=dropout,
param_init_states=True)
core = RecStack(*corelayers)
lastcoredim = core.get_statespec()[-1][0][1][0]
# ATTENTIONS
if mem_attention is None:
mem_attention = Attention(mem_att_dist)
if inp_attention is None:
inp_attention = Attention(inp_att_dist)
if write_addr_generator is None:
write_addr_generator = AttGen(write_addr_dist)
# WRITE VALUE
if write_value_generator is None:
write_value_generator = WriteValGenerator(write_value_dim,
memvocsize,
dropout=dropout)
# MEMORY SAMPLER
if memsampler is not None:
assert (memsamplemethod is None)
if memsamplemethod is not None:
assert (memsampler is None)
memsampler = GumbelSoftmax(temperature=memsampletemp)
################ STATE INTERFACES #################
if not explicit_interface:
if inp_addr_extractor is None:
inp_addr_extractor = Forward(lastcoredim,
lastinpdim + xtra_dim,
dropout=dropout)
if mem_addr_extractor is None:
inp_addr_extractor = Forward(lastcoredim,
lastmemdim + xtra_dim,
dropout=dropout)
# WRITE INTERFACE
if write_addr_extractor is None:
write_addr_extractor = Forward(lastcoredim,
lastmemdim + xtra_dim,
dropout=dropout)
if write_value_extractor is None:
write_value_extractor = Forward(lastcoredim,
write_value_dim,
dropout=dropout)
# MEM UPDATE INTERFACE
if mem_erase_generator is None:
mem_erase_generator = StateToScalar(lastcoredim, scalaraggdim)
if mem_change_generator is None:
mem_change_generator = StateToScalar(lastcoredim, scalaraggdim)
else:
inp_addr_extractor, mem_addr_extractor, write_addr_extractor, \
write_value_extractor, mem_erase_generator, mem_change_generator = \
make_vector_slicers(0, lastinpdim + xtra_dim, lastmemdim + xtra_dim,
lastmemdim + xtra_dim, write_value_dim, 1, 1)
super(SimpleBulkNN,
self).__init__(inpencoder=inpencoder,
memembmat=memembmat,
memencoder=memencoder,
inp_attention=inp_attention,
mem_attention=mem_attention,
core=core,
memsampler=memsampler,
nsteps=nsteps,
inp_addr_extractor=inp_addr_extractor,
mem_addr_extractor=mem_addr_extractor,
write_addr_extractor=write_addr_extractor,
write_addr_generator=write_addr_generator,
mem_erase_generator=mem_erase_generator,
mem_change_generator=mem_change_generator,
write_value_generator=write_value_generator,
write_value_extractor=write_value_extractor,
inp_pos_repr=inp_pos_repr,
mem_pos_repr=mem_pos_repr,
**kw)
def _getattention(self, attdist, sepatt=False):
attgen = AttGen(attdist)
attcon = WeightedSumAttCon()
attention = Attention(attgen, attcon, separate=sepatt)
return attention
