def __init__(self, encdim, invocsize, outvocsize, innerdim, seqlen, **kw):
super(idx2seqStupid, self).__init__(**kw)
self.encdim = encdim
self.invocsize = invocsize
self.outvocsize = outvocsize
self.innerdim = innerdim
self.seqlen = seqlen
self.emb = VectorEmbed(indim=self.invocsize, dim=self.encdim, normalize=True)
self.aletter = stack(Lin(indim=self.encdim, dim=self.outvocsize), Softmax())
self.bletter = stack(Lin(indim=self.encdim, dim=self.outvocsize), Softmax())
self.cletter = stack(Lin(indim=self.encdim, dim=self.outvocsize), Softmax())
def init(self):
#memory
wencpg = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload, self.memdata, indim=self.outdim, outdim=self.encinnerdim+self.entembdim)
#encoder
wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath)
self.enc = SeqEncoder(
wencpg2,
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
#decoder
entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(self.memaddr(self.memblock, indim=self.decinnerdim, memdim=self.memblock.outdim, attdim=self.attdim), Softmax())
self.dec = SeqDecoder(
[entemb2, #self.memblock,
GRU(dim=entemb.dim + self.encinnerdim, innerdim=self.decinnerdim), # GRU(dim=self.memblock.outdim + self.encinnerdim, innerdim=self.decinnerdim),
],
inconcat=True,
innerdim=self.decinnerdim,
softmaxoutblock=self.softmaxoutblock
)
def __init__(self, memblock, memaddr, memattdim=100, indim=None, outnorm=Softmax(), **kw):
super(MemoryStack, self).__init__(**kw)
if not isinstance(memblock, MemoryBlock):
raise Exception("must provide a loaded memory block")
memdim = memblock.outdim
indim = memdim if indim is None else indim
self.exe = stack(memaddr(memblock, memdim=memdim, indim=indim, attdim=memattdim), outnorm)
def __init__(self, memblock, memaddr, memattdim=100, indim=None, outnorm=Softmax(), **kw):
super(MemoryStack, self).__init__(**kw)
if not isinstance(memblock, MemoryBlock):
raise Exception("must provide a loaded memory block")
memdim = memblock.outdim
indim = memdim if indim is None else indim
self.exe = stack(memaddr(memblock, memdim=memdim, indim=indim, attdim=memattdim), outnorm)
def __init__(self, encdim, invocsize, outvocsize, innerdim, seqlen, **kw):
super(idx2seqStupid, self).__init__(**kw)
self.encdim = encdim
self.invocsize = invocsize
self.outvocsize = outvocsize
self.innerdim = innerdim
self.seqlen = seqlen
self.emb = VectorEmbed(indim=self.invocsize,
dim=self.encdim,
normalize=True)
self.aletter = stack(Lin(indim=self.encdim, dim=self.outvocsize),
Softmax())
self.bletter = stack(Lin(indim=self.encdim, dim=self.outvocsize),
Softmax())
self.cletter = stack(Lin(indim=self.encdim, dim=self.outvocsize),
Softmax())
def init(self):
#memory
wencpg = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload, self.memdata, indim=self.outdim, outdim=self.encinnerdim+self.entembdim)
#encoder
wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath)
self.enc = SeqEncoder(
wencpg2,
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
#decoder
entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(self.memaddr(self.memblock, indim=self.decinnerdim, memdim=self.memblock.outdim, attdim=self.attdim), Softmax())
self.dec = SeqDecoder(
[entemb2, #self.memblock,
GRU(dim=entemb.outdim + self.encinnerdim, innerdim=self.decinnerdim), # GRU(dim=self.memblock.outdim + self.encinnerdim, innerdim=self.decinnerdim),
],
inconcat=True,
innerdim=self.decinnerdim,
softmaxoutblock=self.softmaxoutblock
)
def setUp(self):
dim = 50
self.vocabsize = 2000
data = np.arange(0, self.vocabsize).astype("int32")
self.O = param((dim, self.vocabsize)).uniform()
self.W = VectorEmbed(indim=self.vocabsize, dim=50)
self.out = stack(self.W, asblock(lambda x: T.dot(self.O, x)),
Softmax())(Input(ndim=1, dtype="int32"))
def setUp(self):
dim=50
self.vocabsize=2000
data = np.arange(0, self.vocabsize).astype("int32")
self.O = param((dim, self.vocabsize)).uniform()
self.W = VectorEmbed(indim=self.vocabsize, dim=50)
self.out = stack(self.W,
asblock(lambda x: T.dot(self.O, x)),
Softmax())(Input(ndim=1, dtype="int32"))
def setUp(self):
self.dims = [50, 20, 30, 40]
grus = [
GRU(dim=self.dims[i], innerdim=self.dims[i + 1])
for i in range(len(self.dims) - 1)
]
self.s = stack(*grus)
self.paramnames = [
"um", "wm", "uhf", "whf", "u", "w", "bm", "bhf", "b"
]
def __init__(self, outlayers, **kw):
super(Vec2Idx, self).__init__(**kw)
if isinstance(outlayers, MemoryStack):
out = outlayers
else:
if not issequence(outlayers):
outlayers = [outlayers]
if type(outlayers[-1]) is not Softmax:
outlayers.append(Softmax())
out = stack(*outlayers)
self.out = out
def __init__(self, outlayers, **kw):
super(Vec2Idx, self).__init__(**kw)
if isinstance(outlayers, MemoryStack):
out = outlayers
else:
if not issequence(outlayers):
outlayers = [outlayers]
if type(outlayers[-1]) is not Softmax:
outlayers.append(Softmax())
out = stack(*outlayers)
self.out = out
def init(self):
#MEMORY: encodes how entity is written + custom entity embeddings
wencpg = WordEncoderPlusGlove(numchars=self.numchars,
numwords=self.numwords,
encdim=self.wordencdim,
embdim=self.wordembdim,
embtrainfrac=0.0,
glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload,
self.memdata,
indim=self.outdim,
outdim=self.encinnerdim + self.entembdim)
#ENCODER: uses the same language encoder as memory
#wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath)
self.enc = RecStack(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim,
innerdim=self.encinnerdim))
#ATTENTION
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim +
self.decinnerdim,
innerdim=self.attdim)
attcon = WeightedSumAttCon()
#DECODER
#entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(
self.memaddr(self.memblock,
indim=self.decinnerdim + self.encinnerdim,
memdim=self.memblock.outdim,
attdim=self.attdim), Softmax())
self.dec = SeqDecoder([
self.memblock,
GRU(dim=self.entembdim + self.encinnerdim,
innerdim=self.decinnerdim)
],
outconcat=True,
inconcat=False,
attention=Attention(attgen, attcon),
innerdim=self.decinnerdim + self.encinnerdim,
softmaxoutblock=self.softmaxoutblock)
def init(self):
#MEMORY: encodes how entity is written + custom entity embeddings
wencpg = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath)
self.memenco = SeqEncoder(
wencpg,
GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)
)
entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.mempayload = ConcatBlock(entemb, self.memenco)
self.memblock = MemoryBlock(self.mempayload, self.memdata, indim=self.outdim, outdim=self.encinnerdim+self.entembdim)
#ENCODER: uses the same language encoder as memory
#wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath)
self.enc = RecStack(wencpg, GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim))
#ATTENTION
attgen = LinearGateAttentionGenerator(indim=self.encinnerdim + self.decinnerdim, innerdim=self.attdim)
attcon = WeightedSumAttCon()
#DECODER
#entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim)
self.softmaxoutblock = stack(
self.memaddr(
self.memblock,
indim=self.decinnerdim + self.encinnerdim,
memdim=self.memblock.outdim,
attdim=self.attdim),
Softmax())
self.dec = SeqDecoder(
[self.memblock, GRU(dim=self.entembdim + self.encinnerdim, innerdim=self.decinnerdim)],
outconcat=True, inconcat=False,
attention=Attention(attgen, attcon),
innerdim=self.decinnerdim + self.encinnerdim,
softmaxoutblock=self.softmaxoutblock
)
def test_multilevel_set_lr(self):
l1 = Linear(10, 11)
l2 = Linear(11, 12)
l3 = Linear(12, 13)
s = stack(l1, l2, l3)
s[1].set_lr(0.5)
s[2].set_lr(0.1)
o = s(Val(0))
l1o = s[0](Val(0))
l2o = s[1](Val(0))
l3o = s[2](Val(0))
print["{}: {}".format(x, x.lrmul) for x in o.allparams]
for x in o.allparams:
if x in l1o.allparams:
self.assertEqual(x.lrmul, 1.0)
elif x in l2o.allparams:
self.assertEqual(x.lrmul, 0.5)
elif x in l3o.allparams:
self.assertEqual(x.lrmul, 0.1)
s.set_lr(0.21)
o = s(Val(0))
print["{}: {}".format(x, x.lrmul) for x in o.allparams]
for x in o.allparams:
self.assertEqual(x.lrmul, 0.21)
def setUp(self):
self.dims = [50, 20, 30, 40]
grus = [GRU(dim=self.dims[i], innerdim=self.dims[i+1]) for i in range(len(self.dims)-1)]
self.s = stack(*grus)
self.paramnames = ["um", "wm", "uhf", "whf", "u", "w", "bm", "bhf", "b"]
