def _compute_edge_context_3d2(self, vert_factor, edge_factor, reuse=False):
"""
attention-based edge message pooling
this output will be the input edge_ctx of _edge_rnn_forward(edge_ctx, ...
edge_factor and vert_factor are gru_cell
"""
geo_pairs = utils.gather_vec_pairs(
self.geo_state, self.relations
) #utils.gather_vec_pairs(self.geo_state, self.relations)
self.geo_pairs = geo_pairs
sub_geo, obj_geo = tf.split(split_dim=1, num_split=2, value=geo_pairs)
vert_pairs = utils.gather_vec_pairs(
vert_factor, self.relations
) #Tensor("Reshape:0", shape=(?, 1024), dtype=float32) if relation is [24 2], vert_pairs is [24 2*512] so I guess the corresponding object feature for each relation
sub_vert, obj_vert = tf.split(
split_dim=1, num_split=2, value=vert_pairs
) # I guess putting the in (subject) on one part and the out (object) on the other part, because subject verb object, so each is 24 z 512
sub_vert_w_input = tf.concat(concat_dim=1,
values=[sub_vert, edge_factor])
obj_vert_w_input = tf.concat(concat_dim=1,
values=[obj_vert, edge_factor])
# compute compatibility scores
(self.feed(sub_vert_w_input).fc(
1, relu=False, reuse=reuse,
name='sub_vert_w_fc').sigmoid(name='sub_vert_score')
) # passing the input messages from the relation to a fc layer
(self.feed(obj_vert_w_input).fc(
1, relu=False, reuse=reuse,
name='obj_vert_w_fc').sigmoid(name='obj_vert_score')
) # passing the previous state of the object node to a fc layer
# encoding the geomtery of the relation
geo_pairs = self.norm_geo_pairs(geo_pairs, sc=1000)
(self.feed(geo_pairs).fc(
100, relu=False, reuse=reuse,
name='geo_fc1_obj').sigmoid(name='geo_encoded_pre').fc(
512, relu=False, reuse=reuse,
name='geo_fc2_obj').sigmoid(name='geo_encoded')
) # passing the previous state of the object node to a fc layer
geo_encoded = self.get_output('geo_encoded')
# then concatenating wiht the relation feature (edge_factor) and getting a weight out of that
geo_encoded_w_input = tf.concat(concat_dim=1,
values=[geo_encoded, edge_factor])
(self.feed(geo_encoded_w_input).fc(
1, relu=False, reuse=reuse,
name='geo_w_fc').sigmoid(name='geo_vert_score')
) # passing the previous state of the object node to a fc layer
sub_vert_w = self.get_output(
'sub_vert_score'
) #this guy is 24 x 1, and the relations vector is 24,which count for the 4 objects present in the 2 images.
obj_vert_w = self.get_output('obj_vert_score')
geo_vert_w = self.get_output('geo_vert_score')
weighted_sub = tf.mul(sub_vert, sub_vert_w)
weighted_obj = tf.mul(obj_vert, obj_vert_w)
weighted_geo = tf.mul(geo_encoded, geo_vert_w)
return weighted_sub + weighted_obj + weighted_geo
def _compute_edge_context_soft(self,
vert_factor,
edge_factor,
reuse=False):
"""
attention-based edge message pooling
"""
vert_pairs = utils.gather_vec_pairs(vert_factor, self.relations)
sub_vert, obj_vert = tf.split(vert_pairs, num_or_size_splits=2, axis=1)
sub_vert_w_input = tf.concat([sub_vert, edge_factor], 1)
obj_vert_w_input = tf.concat([obj_vert, edge_factor], 1)
# compute compatibility scores
(self.feed(sub_vert_w_input).fc(
1, relu=False, reuse=reuse,
name='sub_vert_w_fc').sigmoid(name='sub_vert_score'))
(self.feed(obj_vert_w_input).fc(
1, relu=False, reuse=reuse,
name='obj_vert_w_fc').sigmoid(name='obj_vert_score'))
sub_vert_w = self.get_output('sub_vert_score')
obj_vert_w = self.get_output('obj_vert_score')
weighted_sub = tf.multiply(sub_vert, sub_vert_w)
weighted_obj = tf.multiply(obj_vert, obj_vert_w)
return weighted_sub + weighted_obj
def _compute_edge_context_2d(self, vert_factor, edge_factor, reuse=False):
"""
attention-based edge message pooling
this output will be the input edge_ctx of _edge_rnn_forward(edge_ctx, ...
edge_factor and vert_factor are gru_cell
"""
vert_pairs = utils.gather_vec_pairs(
vert_factor, self.relations
) #Tensor("Reshape:0", shape=(?, 1024), dtype=float32) if relation is [24 2], vert_pairs is [24 2*512] so I guess the corresponding object feature for each relation
vis_geo = tf.concat(concat_dim=1, values=[vert_pairs, self.geo_state])
# encoding the geomtery of the relation
(self.feed(vis_geo).fc(512,
relu=False,
reuse=reuse,
name='geo_fc2_obj').sigmoid(name='geo_encoded')
) # passing the previous state of the object node to a fc layer
geo_encoded = self.get_output('geo_encoded')
return geo_encoded
def _compute_edge_context_geo_only_stand(self,
vert_factor,
edge_factor,
reuse=False):
"""
same as _compute_edge_context_geo_only, but the normalisation is added.
"""
geo_pairs = utils.gather_vec_pairs(
self.geo_state, self.relations
) #utils.gather_vec_pairs(self.geo_state, self.relations)
geo_pairs = self.norm_geo_pairs(geo_pairs, sc=500)
self.geo_pairs = geo_pairs
# compute compatibility scores
(self.feed(geo_pairs).fc(
100, relu=False, reuse=reuse,
name='geo_fc1').sigmoid(name='geo_score_pre').fc(
512, relu=False, reuse=reuse,
name='geo_fc2').sigmoid(name='geo_score')
) # passing the input messages from the relation to a fc layer
geo_state_w = self.get_output('geo_score')
return geo_state_w
def _compute_edge_context_geo_only(self,
vert_factor,
edge_factor,
reuse=False):
"""
The message will only contains the geometric information
"""
geo_pairs = utils.gather_vec_pairs(
self.geo_state, self.relations
) #utils.gather_vec_pairs(self.geo_state, self.relations)
self.geo_pairs = geo_pairs
# compute compatibility scores
(self.feed(geo_pairs).fc(
100, relu=False, reuse=reuse,
name='geo_fc1').sigmoid(name='geo_score_pre').fc(
512, relu=False, reuse=reuse,
name='geo_fc2').sigmoid(name='geo_score')
) # passing the input messages from the relation to a fc layer
geo_state_w = self.get_output('geo_score')
return geo_state_w
def _compute_edge_context_soft(self,
vert_factor,
edge_factor,
reuse=False):
"""
attention-based edge message pooling
this output will be the input edge_ctx of _edge_rnn_forward(edge_ctx, ...
edge_factor and vert_factor are gru_cell
"""
vert_pairs = utils.gather_vec_pairs(
vert_factor, self.relations
) #Tensor("Reshape:0", shape=(?, 1024), dtype=float32) if relation is [24 2], vert_pairs is [24 2 512] so I guess the corresponding object feature for each relation
sub_vert, obj_vert = tf.split(
split_dim=1, num_split=2, value=vert_pairs
) # I guess putting the in (subject) on one part and the out (object) on the other part, because subject verb object
sub_vert_w_input = tf.concat(
concat_dim=1,
values=[sub_vert, edge_factor
]) # sub_vert with the state of the predicate cell at tm1
obj_vert_w_input = tf.concat(concat_dim=1,
values=[obj_vert,
edge_factor]) # 24 x 1024
# compute compatibility scores
(self.feed(sub_vert_w_input).fc(
1, relu=False, reuse=reuse,
name='sub_vert_w_fc').sigmoid(name='sub_vert_score')
) # passing the input messages from the relation to a fc layer
(self.feed(obj_vert_w_input).fc(
1, relu=False, reuse=reuse,
name='obj_vert_w_fc').sigmoid(name='obj_vert_score')
) # passing the previous state of the object node to a fc layer
sub_vert_w = self.get_output('sub_vert_score')
obj_vert_w = self.get_output('obj_vert_score')
weighted_sub = tf.mul(sub_vert, sub_vert_w)
weighted_obj = tf.mul(obj_vert, obj_vert_w)
return weighted_sub + weighted_obj