def baselineAttLayer(self,
images,
memory,
inDim,
hDim,
name="",
reuse=None):
with tf.variable_scope("attLayer" + name, reuse=reuse):
# projImages = ops.linear(images, inDim, hDim, name = "projImage")
# projMemory = tf.expand_dims(ops.linear(memory, inDim, hDim, name = "projMemory"), axis = -2)
# if config.saMultiplicative:
# interactions = projImages * projMemory
# else:
# interactions = tf.tanh(projImages + projMemory)
interactions, _ = ops.mul(images,
memory,
inDim,
proj={
"dim": hDim,
"shared": False
},
interMod=config.baselineAttType)
attention = ops.inter2att(interactions, hDim)
summary = ops.att2Smry(attention, images)
newMemory = memory + summary
return newMemory
def baseline(self, vecQuestions, questionDim, images, imageDim, hDim):
with tf.variable_scope("baseline"):
if config.baselineAtt:
memory = ops.linear(vecQuestions,
questionDim,
hDim,
name="qProj")
images = ops.linear(images, imageDim, hDim, name="iProj")
for i in range(config.baselineAttNumLayers):
memory = self.baselineAttLayer(images,
memory,
hDim,
hDim,
name="baseline%d" % i)
memDim = hDim
else:
if config.imageObjects:
cff = tf.get_variable(
"cff",
shape=(imageDim, ),
initializer=tf.random_normal_initializer())
interactions, hDim = ops.mul(images, cff, imageDim)
attention = ops.inter2att(interactions,
hDim,
mask=self.imagesObjectNum)
images = ops.att2Smry(attention, images)
else:
images, imageDim = ops.linearizeFeatures(
images,
self.H,
self.W,
imageDim,
projDim=config.baselineProjDim)
if config.baselineLSTM and config.baselineCNN:
memory = tf.concat([vecQuestions, images], axis=-1)
memDim = questionDim + imageDim
elif config.baselineLSTM:
memory = vecQuestions
memDim = questionDim
else: # config.baselineCNN
memory = images
memDim = imageDim
return memory, memDim
def memAutoEnc(newMemory, info, control, name="", reuse=None):
with tf.variable_scope("memAutoEnc" + name, reuse=reuse):
# inputs to auto encoder
features = info if config.autoEncMemInputs == "INFO" else newMemory
features = ops.linear(features,
config.memDim,
config.ctrlDim,
act=config.autoEncMemAct,
name="aeMem")
# reconstruct control
if config.autoEncMemLoss == "CONT":
loss = tf.reduce_mean(tf.squared_difference(control, features))
else:
interactions, dim = ops.mul(
self.questionCntxWords,
features,
config.ctrlDim,
concat={"x": config.autoEncMemCnct},
mulBias=config.mulBias,
name="aeMem")
logits = ops.inter2logits(interactions, dim)
logits = self.expMask(logits, self.questionLengths)
# reconstruct word attentions
if config.autoEncMemLoss == "PROB":
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(
labels=self.attentions["question"][-1],
logits=logits))
# reconstruct control through words attentions
else:
attention = tf.nn.softmax(logits)
summary = ops.att2Smry(attention, self.questionCntxWords)
loss = tf.reduce_mean(
tf.squared_difference(control, summary))
return loss
def write(self,
memory,
info,
control,
contControl=None,
name="",
reuse=None):
with tf.variable_scope("write" + name, reuse=reuse):
# optionally project info
if config.writeInfoProj:
info = ops.linear(info,
config.memDim,
config.memDim,
name="info")
# optional info nonlinearity
info = ops.activations[config.writeInfoAct](info)
# compute self-attention vector based on previous controls and memories
if config.writeSelfAtt:
print("using self attention")
selfControl = control
if config.writeSelfAttMod == "CONT":
selfControl = contControl
# elif config.writeSelfAttMod == "POST":
# selfControl = postControl
selfControl = ops.linear(selfControl,
config.ctrlDim,
config.ctrlDim,
name="ctrlProj")
interactions = self.controls * tf.expand_dims(selfControl,
axis=1)
# if config.selfAttShareInter:
# selfAttlogits = self.linearP(selfAttInter, config.encDim, 1, self.interL[0], self.interL[1], name = "modSelfAttInter")
attention = ops.inter2att(interactions,
config.ctrlDim,
name="selfAttention")
self.attentions["self"].append(attention)
selfSmry = ops.att2Smry(attention, self.memories)
# get write unit inputs: previous memory, the new info, optionally self-attention / control
newMemory, dim = memory, config.memDim
if config.writeInputs == "INFO":
newMemory = info
elif config.writeInputs == "SUM":
newMemory += info
elif config.writeInputs == "BOTH":
newMemory, dim = ops.concat(newMemory,
info,
dim,
mul=config.writeConcatMul)
# else: MEM
if config.writeSelfAtt:
newMemory = tf.concat([newMemory, selfSmry], axis=-1)
dim += config.memDim
if config.writeMergeCtrl:
newMemory = tf.concat([newMemory, control], axis=-1)
dim += config.memDim
# project memory back to memory dimension
if config.writeMemProj or (dim != config.memDim):
newMemory = ops.linear(newMemory,
dim,
config.memDim,
name="newMemory")
# optional memory nonlinearity
newMemory = ops.activations[config.writeMemAct](newMemory)
# write unit gate
if config.writeGate:
gateDim = config.memDim
if config.writeGateShared:
gateDim = 1
z = tf.sigmoid(
ops.linear(control,
config.ctrlDim,
gateDim,
name="gate",
bias=config.writeGateBias))
self.attentions["gate"].append(z)
newMemory = newMemory * z + memory * (1 - z)
# optional batch normalization
if config.memoryBN:
newMemory = tf.contrib.layers.batch_norm(
newMemory,
decay=config.bnDecay,
center=config.bnCenter,
scale=config.bnScale,
is_training=self.train,
updates_collections=None)
return newMemory
def control(self,
controlInput,
inWords,
outWords,
questionLengths,
control,
contControl=None,
name="",
reuse=None):
with tf.variable_scope("control" + name, reuse=reuse):
dim = config.ctrlDim
## Step 1: compute "continuous" control state given previous control and question.
# control inputs: question and previous control
newContControl = controlInput
if config.controlFeedPrev:
newContControl = control if config.controlFeedPrevAtt else contControl
if config.controlFeedInputs:
newContControl = tf.concat([newContControl, controlInput],
axis=-1)
dim += config.ctrlDim
# merge inputs together
newContControl = ops.linear(newContControl,
dim,
config.ctrlDim,
act=config.controlContAct,
name="contControl")
dim = config.ctrlDim
## Step 2: compute attention distribution over words and sum them up accordingly.
# compute interactions with question words
interactions = tf.expand_dims(newContControl, axis=1) * inWords
# optionally concatenate words
if config.controlConcatWords:
interactions = tf.concat([interactions, inWords], axis=-1)
dim += config.ctrlDim
# optional projection
if config.controlProj:
interactions = ops.linear(interactions,
dim,
config.ctrlDim,
act=config.controlProjAct)
dim = config.ctrlDim
# compute attention distribution over words and summarize them accordingly
logits = ops.inter2logits(interactions, dim)
# self.interL = (interW, interb)
# if config.controlCoverage:
# logits += coverageBias * coverage
attention = tf.nn.softmax(ops.expMask(logits, questionLengths))
self.attentions["question"].append(attention)
# if config.controlCoverage:
# coverage += attention # Add logits instead?
newControl = ops.att2Smry(attention, outWords)
# ablation: use continuous control (pre-attention) instead
if config.controlContinuous:
newControl = newContControl
return newControl, newContControl
def read(self, knowledgeBase, memory, control, name="", reuse=None):
with tf.variable_scope("read" + name, reuse=reuse):
dim = config.memDim
## memory dropout
if config.memoryVariationalDropout:
memory = ops.applyVarDpMask(memory, self.memDpMask,
self.dropouts["memory"])
else:
memory = tf.nn.dropout(memory, self.dropouts["memory"])
## Step 1: knowledge base / memory interactions
# parameters for knowledge base and memory projection
proj = None
if config.readProjInputs:
proj = {
"dim": config.attDim,
"shared": config.readProjShared,
"dropout": self.dropouts["read"]
}
dim = config.attDim
# parameters for concatenating knowledge base elements
concat = {
"x": config.readMemConcatKB,
"proj": config.readMemConcatProj
}
# compute interactions between knowledge base and memory
interactions, interDim = ops.mul(x=knowledgeBase,
y=memory,
dim=config.memDim,
proj=proj,
concat=concat,
interMod=config.readMemAttType,
name="memInter")
projectedKB = proj.get("x") if proj else None
# project memory interactions back to hidden dimension
if config.readMemProj:
interactions = ops.linear(interactions,
interDim,
dim,
act=config.readMemAct,
name="memKbProj")
else:
dim = interDim
## Step 2: compute interactions with control
if config.readCtrl:
# compute interactions with control
if config.ctrlDim != dim:
control = ops.linear(control,
ctrlDim,
dim,
name="ctrlProj")
interactions, interDim = ops.mul(
interactions,
control,
dim,
interMod=config.readCtrlAttType,
concat={"x": config.readCtrlConcatInter},
name="ctrlInter")
# optionally concatenate knowledge base elements
if config.readCtrlConcatKB:
if config.readCtrlConcatProj:
addedInp, addedDim = projectedKB, config.attDim
else:
addedInp, addedDim = knowledgeBase, config.memDim
interactions = tf.concat([interactions, addedInp], axis=-1)
dim += addedDim
# optional nonlinearity
interactions = ops.activations[config.readCtrlAct](
interactions)
## Step 3: sum attentions up over the knowledge base
# transform vectors to attention distribution
attention = ops.inter2att(interactions,
dim,
dropout=self.dropouts["read"])
self.attentions["kb"].append(attention)
# optionally use projected knowledge base instead of original
if config.readSmryKBProj:
knowledgeBase = projectedKB
# sum up the knowledge base according to the distribution
information = ops.att2Smry(attention, knowledgeBase)
return information
