def train_gnn(input_graphs,target_graphs):
iterations = 1000
learning_rate = 1e-3
optimizer = snt.optimizers.Adam(learning_rate)
model = models.EncodeProcessDecode(node_output_size=7)
def create_loss(target, outputs):
loss = [
tf.compat.v1.losses.mean_squared_error(target.nodes, output.nodes)
for output in outputs
]
return tf.stack(loss)
def update_step(inputs_tr, targets_tr):
with tf.GradientTape() as tape:
outputs_tr = model(inputs_tr,10)
loss_tr = create_loss(targets_tr, outputs_tr)
loss_tr = tf.math.reduce_sum(loss_tr) / 10
gradients = tape.gradient(loss_tr, model.trainable_variables)
optimizer.apply(gradients, model.trainable_variables)
return outputs_tr, loss_tr
for iteration in range(iterations):
inputs_tr = utils_tf.get_graph(input_graphs, slice(0, 80))
targets_tr = utils_tf.get_graph(target_graphs, slice(0, 80))
outputs_tr, loss_tr = update_step(inputs_tr, targets_tr)
print(loss_tr)
print(outputs_tr)
def test_raises(self, index, error_type, message, use_constant, use_slice):
if use_constant:
index = tf.constant(index)
if use_slice:
index = slice(index)
graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(self.graphs_dicts_in)
with self.assertRaisesRegexp(error_type, message):
utils_tf.get_graph(graphs_tuple, index)
def create_linked_list_target(batch_size, input_graphs):
"""Creates linked list targets.
Returns a graph with the same number of nodes as `input_graph`. Each node
contains a 2d vector with targets for a 1-class classification where only one
node is `True`, the smallest value in the array. The vector contains two
values: [prob_true, prob_false].
It also contains edges connecting all nodes. These are again 2d vectors with
softmax targets [prob_true, prob_false]. An edge is True
if n+1 is the element immediately after n in the sorted list.
Args:
batch_size: batch size for the `input_graphs`.
input_graphs: a `graphs.GraphsTuple` which contains a batch of inputs.
Returns:
A `graphs.GraphsTuple` with the targets, which encode the linked list.
"""
target_graphs = []
for i in range(batch_size):
input_graph = utils_tf.get_graph(input_graphs, i)
num_elements = tf.shape(input_graph.nodes)[0]
si = tf.cast(tf.squeeze(input_graph.nodes), tf.int32)
nodes = tf.reshape(tf.one_hot(si[:1], num_elements), (-1, 1))
x = tf.stack((si[:-1], si[1:]))[None]
y = tf.stack((input_graph.senders, input_graph.receivers),
axis=1)[:, :, None]
edges = tf.reshape(
tf.cast(
tf.reduce_any(tf.reduce_all(tf.equal(x, y), axis=1), axis=1),
tf.float32), (-1, 1))
target_graphs.append(input_graph._replace(nodes=nodes, edges=edges))
return utils_tf.concat(target_graphs, axis=0)
def equal_graphs(graphs_a, graphs_b, verbose=True):
cond = tf.constant([True])
v_a = utils_tf.get_num_graphs(graphs_a)
v_b = utils_tf.get_num_graphs(graphs_b)
cond = tf.math.logical_and(cond, v_a == v_b)
if not cond and verbose:
pprint("num_graphs", cond.numpy())
for field in [
"n_node",
"n_edge",
"globals",
"nodes",
"edges",
"receivers",
"senders",
]:
v_a = getattr(graphs_a, field)
v_b = getattr(graphs_b, field)
check_values = tf.reduce_all(tf.equal(v_a, v_b))
cond = tf.math.logical_and(cond, check_values)
check_shape = tf.reduce_all(tf.equal(v_a.shape, v_b.shape))
cond = tf.math.logical_and(cond, check_shape)
if not cond and verbose:
pprint(
field,
f"values = {check_values.numpy()}",
f"shape = {check_shape.numpy()}",
cond.numpy(),
)
if verbose:
print()
if utils_tf.get_num_graphs(graphs_a) > 1:
for graph_idx in range(utils_tf.get_num_graphs(graphs_a)):
graph_a = utils_tf.get_graph(graphs_a, graph_idx)
graph_b = utils_tf.get_graph(graphs_b, graph_idx)
check = equal_graphs(graph_a, graph_b, verbose=False)
cond = tf.math.logical_and(cond, check)
return cond
def test_getitem_one(self, use_tensor_index):
index = 2
expected = self.graphs_dicts_out[index]
if use_tensor_index:
index = tf.constant(index)
graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(self.graphs_dicts_in)
graph = utils_tf.get_graph(graphs_tuple, index)
graph = utils_tf.nest_to_numpy(graph)
actual, = utils_np.graphs_tuple_to_data_dicts(graph)
for k, v in expected.items():
self.assertAllClose(v, actual[k])
self.assertEqual(expected["nodes"].shape[0], actual["n_node"])
self.assertEqual(expected["edges"].shape[0], actual["n_edge"])
def test_getitem_one(self):
index = 2
expected = self.graphs_dicts_out[index]
graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(
self.graphs_dicts_in)
graph_op = utils_tf.get_graph(graphs_tuple, index)
graph_op = utils_tf.make_runnable_in_session(graph_op)
with self.test_session() as sess:
graph = sess.run(graph_op)
actual, = utils_np.graphs_tuple_to_data_dicts(graph)
for k, v in expected.items():
self.assertAllClose(v, actual[k])
self.assertEqual(expected["nodes"].shape[0], actual["n_node"])
self.assertEqual(expected["edges"].shape[0], actual["n_edge"])
def test_getitem(self, use_tensor_slice):
index = slice(1, 3)
expected = self.graphs_dicts_out[index]
if use_tensor_slice:
index = slice(tf.constant(index.start), tf.constant(index.stop))
graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(self.graphs_dicts_in)
graphs2 = utils_tf.get_graph(graphs_tuple, index)
graphs2 = utils_tf.nest_to_numpy(graphs2)
actual = utils_np.graphs_tuple_to_data_dicts(graphs2)
for ex, ac in zip(expected, actual):
for k, v in ex.items():
self.assertAllClose(v, ac[k])
self.assertEqual(ex["nodes"].shape[0], ac["n_node"])
self.assertEqual(ex["edges"].shape[0], ac["n_edge"])
def test_getitem(self):
index = slice(1, 3)
expected = self.graphs_dicts_out[index]
graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(
self.graphs_dicts_in)
graphs2_op = utils_tf.get_graph(graphs_tuple, index)
graphs2_op = utils_tf.make_runnable_in_session(graphs2_op)
with self.test_session() as sess:
graphs2 = sess.run(graphs2_op)
actual = utils_np.graphs_tuple_to_data_dicts(graphs2)
for ex, ac in zip(expected, actual):
for k, v in ex.items():
self.assertAllClose(v, ac[k])
self.assertEqual(ex["nodes"].shape[0], ac["n_node"])
self.assertEqual(ex["edges"].shape[0], ac["n_edge"])
def _unpack_graphs_tuple(graphs_tuple, max_n_node, max_n_edge):
"""Unpacks data from a GraphTuple instance.
Args:
graphs_tuple: A GraphTuple instance.
max_n_node: Maximum number of nodes.
max_n_edge: Maximum number of edges.
Returns:
batch_nodes: A [batch, max_n_node, dims] float tensor.
batch_edges: A [batch, max_n_edge, dims] float tensor.
"""
batch_nodes = []
batch_edges = []
batch_size = graphs_tuple.n_node.shape[0]
for i in range(batch_size):
graph_tuple_at_i = utils_tf.get_graph(graphs_tuple, i)
n_node = tf.squeeze(graph_tuple_at_i.n_node)
n_edge = tf.squeeze(graph_tuple_at_i.n_edge)
nodes = graph_tuple_at_i.nodes
edges = graph_tuple_at_i.edges
if not None in [nodes, max_n_node]:
batch_nodes.append(
tf.pad(tensor=nodes,
paddings=[[0, max_n_node - n_node], [0, 0]]))
if not None in [edges, max_n_edge]:
batch_edges.append(
tf.pad(tensor=edges,
paddings=[[0, max_n_edge - n_edge], [0, 0]]))
batch_nodes = tf.stack(batch_nodes, 0) if batch_nodes else None
batch_edges = tf.stack(batch_edges, 0) if batch_edges else None
return batch_nodes, batch_edges
def test_raises(self, index, error_type, message):
graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(self.graphs_dicts_in)
with self.assertRaisesRegexp(error_type, message):
utils_tf.get_graph(graphs_tuple, index)
x = tf.placeholder(shape=[None, features.shape[1]], dtype=tf.float32)
y = tf.placeholder(shape=[None, y_train.shape[1]], dtype=tf.float32)
train_mask_holder = tf.placeholder(shape=[
None,
], dtype=tf.int32)
# define a Graph object
input_graphs.append({"nodes": x})
input_graphs = utils_tf.data_dicts_to_graphs_tuple(input_graphs)
input_graphs = utils_tf.fully_connect_graph_dynamic(input_graphs)
adj = tf.constant(adj.toarray(), dtype=tf.float32)
model = GCN()
input_graph = utils_tf.get_graph(input_graphs, 0)
# the output_graph = make_linear_model()(input_graph)
# the output_graph.nodes = make_linear_model()(input_graph.nodes)
output_graph = model(input_graph)
# Make the graph can be ran in a session
output_graph, input_graph = make_all_runnable_in_session(
output_graph, input_graph)
# Define the loss function
loss = tf.nn.softmax_cross_entropy_with_logits(
labels=tf.boolean_mask(y, train_mask_holder),
logits=tf.boolean_mask(output_graph.nodes, train_mask_holder))
def _parse_single_example(example, options):
"""Parses a single tf.Example proto.
Args:
example: An Example proto.
options: An instance of reader_pb2.Reader.
Returns:
A dictionary indexed by tensor name.
"""
# Initialize `keys_to_features`.
example_fmt = {
'id':
tf.io.FixedLenFeature([], tf.int64, default_value=-1),
# Proposals
'image/n_proposal':
tf.io.FixedLenFeature([], tf.int64, default_value=0),
'image/proposal/bbox/ymin':
tf.io.VarLenFeature(tf.float32),
'image/proposal/bbox/xmin':
tf.io.VarLenFeature(tf.float32),
'image/proposal/bbox/ymax':
tf.io.VarLenFeature(tf.float32),
'image/proposal/bbox/xmax':
tf.io.VarLenFeature(tf.float32),
'image/proposal/feature':
tf.io.VarLenFeature(tf.float32),
# Caption graph.
'caption_graph/caption':
tf.io.VarLenFeature(tf.string),
'caption_graph/n_node':
tf.io.VarLenFeature(tf.int64),
'caption_graph/n_edge':
tf.io.VarLenFeature(tf.int64),
'caption_graph/nodes':
tf.io.VarLenFeature(tf.string),
'caption_graph/edges':
tf.io.VarLenFeature(tf.string),
'caption_graph/senders':
tf.io.VarLenFeature(tf.int64),
'caption_graph/receivers':
tf.io.VarLenFeature(tf.int64),
# Ground-truth.
'scene_graph/n_relation':
tf.io.FixedLenFeature([], tf.int64, default_value=0),
'scene_graph/predicate':
tf.io.VarLenFeature(tf.string),
'scene_graph/subject':
tf.io.VarLenFeature(tf.string),
'scene_graph/subject/bbox/ymin':
tf.io.VarLenFeature(tf.float32),
'scene_graph/subject/bbox/xmin':
tf.io.VarLenFeature(tf.float32),
'scene_graph/subject/bbox/ymax':
tf.io.VarLenFeature(tf.float32),
'scene_graph/subject/bbox/xmax':
tf.io.VarLenFeature(tf.float32),
'scene_graph/object':
tf.io.VarLenFeature(tf.string),
'scene_graph/object/bbox/ymin':
tf.io.VarLenFeature(tf.float32),
'scene_graph/object/bbox/xmin':
tf.io.VarLenFeature(tf.float32),
'scene_graph/object/bbox/ymax':
tf.io.VarLenFeature(tf.float32),
'scene_graph/object/bbox/xmax':
tf.io.VarLenFeature(tf.float32),
}
# Decode proposals.
parsed = tf.parse_single_example(example, example_fmt)
proposals = tfexample_decoder.BoundingBox(
prefix='image/proposal/bbox/').tensors_to_item(parsed)
n_proposal = tf.minimum(parsed['image/n_proposal'], options.max_n_proposal)
proposals = proposals[:options.max_n_proposal, :]
proposal_features = tf.reshape(
tf.sparse_tensor_to_dense(parsed['image/proposal/feature']),
[-1, options.feature_dimensions])[:options.max_n_proposal, :]
# Decode and randomly sample a caption graph.
graphs = GraphsTuple(
globals=None,
n_node=tf.sparse_tensor_to_dense(parsed['caption_graph/n_node'], 0),
n_edge=tf.sparse_tensor_to_dense(parsed['caption_graph/n_edge'], 0),
nodes=tf.sparse_tensor_to_dense(parsed['caption_graph/nodes'], ''),
edges=tf.sparse_tensor_to_dense(parsed['caption_graph/edges'], ''),
senders=tf.sparse_tensor_to_dense(parsed['caption_graph/senders'], 0),
receivers=tf.sparse_tensor_to_dense(parsed['caption_graph/receivers'], 0))
num_graphs = utils_tf.get_num_graphs(graphs)
index = tf.random.uniform([], minval=0, maxval=num_graphs, dtype=tf.int32)
caption = tf.sparse_tensor_to_dense(parsed['caption_graph/caption'])[index]
caption_graph = utils_tf.get_graph(graphs, index)
# Decode the ground-truth.
subject_boxes = tfexample_decoder.BoundingBox(
prefix='scene_graph/subject/bbox/').tensors_to_item(parsed)
object_boxes = tfexample_decoder.BoundingBox(
prefix='scene_graph/object/bbox/').tensors_to_item(parsed)
feature_dict = {
'id':
parsed['id'],
# Proposals.
'image/n_proposal':
n_proposal,
'image/proposal':
proposals,
'image/proposal/feature':
proposal_features,
# Caption graph.
'caption_graph/caption':
caption,
'caption_graph/n_node':
caption_graph.n_node[0],
'caption_graph/n_edge':
caption_graph.n_edge[0],
'caption_graph/nodes':
caption_graph.nodes,
'caption_graph/edges':
caption_graph.edges,
'caption_graph/senders':
caption_graph.senders,
'caption_graph/receivers':
caption_graph.receivers,
# Ground-truth.
'scene_graph/n_relation':
parsed['scene_graph/n_relation'],
'scene_graph/predicate':
tf.sparse_tensor_to_dense(parsed['scene_graph/predicate'], ''),
'scene_graph/subject':
tf.sparse_tensor_to_dense(parsed['scene_graph/subject'], ''),
'scene_graph/subject/box':
subject_boxes,
'scene_graph/object':
tf.sparse_tensor_to_dense(parsed['scene_graph/object'], ''),
'scene_graph/object/box':
object_boxes,
}
for key in feature_dict.keys():
if key != 'id' and feature_dict[key].dtype == tf.int64:
feature_dict[key] = tf.cast(feature_dict[key], tf.int32)
return feature_dict
# define a Graph object
input_graphs.append({"nodes": x,
"edges": edges,
"receivers": receivers,
"senders": senders})
input_graphs = utils_tf.data_dicts_to_graphs_tuple(input_graphs)
model = GCN()
output_graphs = model(input_graphs)
# Make the graph can be ran in a session
# Define the loss function
output_graph = utils_tf.get_graph(output_graphs, 0)
output_graphs, input_graphs, output_graph = make_all_runnable_in_session(
output_graphs, input_graphs, output_graph)
loss = tf.nn.softmax_cross_entropy_with_logits(
labels=tf.boolean_mask(
y, train_mask_holder), logits=tf.boolean_mask(
output_graph.nodes, train_mask_holder))
# define the optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=args.lr)
shape = train_mask.shape[0]
step_op = optimizer.minimize(loss)
sess = tf.Session()
# initialize all the variables in the graph.
sess.run(tf.global_variables_initializer())
