def __call__(self, inp, output_prev, _img):
shared = self.Shared(self.Embed(inp))
img_pred = self.Visual(shared)
txt_pred = softmax3d(
self.Embed.unembed(
self.ToTxt(self.LM(last(shared), self.Embed(output_prev)))))
return (img_pred, txt_pred)
def __call__(self, input):
return self.ToImg(last(self.Encode(input)))
def __call__(self, input):
# Using last because Sum returns the whole seq of partial sums
# to be compatible with recurrent layers.
return util.l2norm(last(self.Encode(input)))
def encode(self, inp):
return last(self.Encode(inp))
def __call__(self, inp):
return self.visual_activation(self.ToImg(last(self.Encode(inp))))
def __call__(self, inp, out_prev, _img):
rep = last(self.Encode(self.Embed(inp)))
img = self.visual_activation(self.DecodeV(rep))
txt = softmax3d(self.Embed.unembed(self.PredictT(self.DecodeT(rep, self.Embed(out_prev)))))
return (img, txt)
def __call__(self, input):
return util.l2norm(last(self.Encode(input)))
def __call__(self, inp):
return self.visual_activation(self.ToImg(last(self.Encode(inp))))
def __call__(self, input):
return self.project(layer.last(self.encode(input)))
def bidi_encode(layer, inp):
"""Return encoded input from a bidirectional encoder."""
f,b = layer.bidi(inp)
return last(f) + first(b)
def __call__(self, input):
rep = last(self.Encode(input))
return self.ToImg(rep)
def __call__(self, inp):
return self.Project(last(self.Encode(inp)))
def predictor_v(self):
"""Return function to predict image vector from input."""
input = T.imatrix()
return theano.function([input], self.visual_activation(self.ToVis(last(self.Encode(self.Embed(input))))))
def __call__(self, inp, out_prev, img):
img_out = self.visual_activation(self.ToVis(last(self.Encode(self.Embed(inp)))))
txt_out = softmax3d(self.Embed.unembed(self.PredictT(self.Decode(self.visual_activation(self.FromVis(img)),
self.Embed(out_prev)))))
return (img_out, txt_out)
def __call__(self, input):
return self.project(layer.last(self.encode(input)))
def predictor_r(self):
"""Return function to predict representation from input."""
input = T.imatrix()
return theano.function([input], last(self.Shared(self.Embed(input))))
def __call__(self, inp, output_prev, _img):
shared = self.Shared(self.Embed(inp))
img_pred = self.Visual(shared)
txt_pred = softmax3d(self.Embed.unembed(self.ToTxt(self.LM(last(shared), self.Embed(output_prev)))))
return (img_pred, txt_pred)
def encode(self, inp):
return last(self.Encode(inp))
def predictor_r(self):
"""Return function to predict representation from input."""
input = T.imatrix()
return theano.function([input], last(self.Shared(self.Embed(input))))
def __call__(self, input):
return self.network.ToImg(last(self.network.EncodeV(self.network.Shared(input))))
def make_predict_next(net):
out_prev = T.imatrix()
rep_prev = T.fmatrix()
rep = net.LM(rep_prev, net.Embed(out_prev))
out = softmax3d(net.Embed.unembed(net.ToTxt(rep)))
return theano.function([rep_prev, out_prev], [last(rep), out])