def train_gnn(input_graphs,target_graphs): iterations = 1000 learning_rate = 1e-3 optimizer = snt.optimizers.Adam(learning_rate) model = models.EncodeProcessDecode(node_output_size=7) def create_loss(target, outputs): loss = [ tf.compat.v1.losses.mean_squared_error(target.nodes, output.nodes) for output in outputs ] return tf.stack(loss) def update_step(inputs_tr, targets_tr): with tf.GradientTape() as tape: outputs_tr = model(inputs_tr,10) loss_tr = create_loss(targets_tr, outputs_tr) loss_tr = tf.math.reduce_sum(loss_tr) / 10 gradients = tape.gradient(loss_tr, model.trainable_variables) optimizer.apply(gradients, model.trainable_variables) return outputs_tr, loss_tr for iteration in range(iterations): inputs_tr = utils_tf.get_graph(input_graphs, slice(0, 80)) targets_tr = utils_tf.get_graph(target_graphs, slice(0, 80)) outputs_tr, loss_tr = update_step(inputs_tr, targets_tr) print(loss_tr) print(outputs_tr)
def test_raises(self, index, error_type, message, use_constant, use_slice): if use_constant: index = tf.constant(index) if use_slice: index = slice(index) graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(self.graphs_dicts_in) with self.assertRaisesRegexp(error_type, message): utils_tf.get_graph(graphs_tuple, index)
def create_linked_list_target(batch_size, input_graphs): """Creates linked list targets. Returns a graph with the same number of nodes as `input_graph`. Each node contains a 2d vector with targets for a 1-class classification where only one node is `True`, the smallest value in the array. The vector contains two values: [prob_true, prob_false]. It also contains edges connecting all nodes. These are again 2d vectors with softmax targets [prob_true, prob_false]. An edge is True if n+1 is the element immediately after n in the sorted list. Args: batch_size: batch size for the `input_graphs`. input_graphs: a `graphs.GraphsTuple` which contains a batch of inputs. Returns: A `graphs.GraphsTuple` with the targets, which encode the linked list. """ target_graphs = [] for i in range(batch_size): input_graph = utils_tf.get_graph(input_graphs, i) num_elements = tf.shape(input_graph.nodes)[0] si = tf.cast(tf.squeeze(input_graph.nodes), tf.int32) nodes = tf.reshape(tf.one_hot(si[:1], num_elements), (-1, 1)) x = tf.stack((si[:-1], si[1:]))[None] y = tf.stack((input_graph.senders, input_graph.receivers), axis=1)[:, :, None] edges = tf.reshape( tf.cast( tf.reduce_any(tf.reduce_all(tf.equal(x, y), axis=1), axis=1), tf.float32), (-1, 1)) target_graphs.append(input_graph._replace(nodes=nodes, edges=edges)) return utils_tf.concat(target_graphs, axis=0)
def equal_graphs(graphs_a, graphs_b, verbose=True): cond = tf.constant([True]) v_a = utils_tf.get_num_graphs(graphs_a) v_b = utils_tf.get_num_graphs(graphs_b) cond = tf.math.logical_and(cond, v_a == v_b) if not cond and verbose: pprint("num_graphs", cond.numpy()) for field in [ "n_node", "n_edge", "globals", "nodes", "edges", "receivers", "senders", ]: v_a = getattr(graphs_a, field) v_b = getattr(graphs_b, field) check_values = tf.reduce_all(tf.equal(v_a, v_b)) cond = tf.math.logical_and(cond, check_values) check_shape = tf.reduce_all(tf.equal(v_a.shape, v_b.shape)) cond = tf.math.logical_and(cond, check_shape) if not cond and verbose: pprint( field, f"values = {check_values.numpy()}", f"shape = {check_shape.numpy()}", cond.numpy(), ) if verbose: print() if utils_tf.get_num_graphs(graphs_a) > 1: for graph_idx in range(utils_tf.get_num_graphs(graphs_a)): graph_a = utils_tf.get_graph(graphs_a, graph_idx) graph_b = utils_tf.get_graph(graphs_b, graph_idx) check = equal_graphs(graph_a, graph_b, verbose=False) cond = tf.math.logical_and(cond, check) return cond
def test_getitem_one(self, use_tensor_index): index = 2 expected = self.graphs_dicts_out[index] if use_tensor_index: index = tf.constant(index) graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(self.graphs_dicts_in) graph = utils_tf.get_graph(graphs_tuple, index) graph = utils_tf.nest_to_numpy(graph) actual, = utils_np.graphs_tuple_to_data_dicts(graph) for k, v in expected.items(): self.assertAllClose(v, actual[k]) self.assertEqual(expected["nodes"].shape[0], actual["n_node"]) self.assertEqual(expected["edges"].shape[0], actual["n_edge"])
def test_getitem_one(self): index = 2 expected = self.graphs_dicts_out[index] graphs_tuple = utils_tf.data_dicts_to_graphs_tuple( self.graphs_dicts_in) graph_op = utils_tf.get_graph(graphs_tuple, index) graph_op = utils_tf.make_runnable_in_session(graph_op) with self.test_session() as sess: graph = sess.run(graph_op) actual, = utils_np.graphs_tuple_to_data_dicts(graph) for k, v in expected.items(): self.assertAllClose(v, actual[k]) self.assertEqual(expected["nodes"].shape[0], actual["n_node"]) self.assertEqual(expected["edges"].shape[0], actual["n_edge"])
def test_getitem(self, use_tensor_slice): index = slice(1, 3) expected = self.graphs_dicts_out[index] if use_tensor_slice: index = slice(tf.constant(index.start), tf.constant(index.stop)) graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(self.graphs_dicts_in) graphs2 = utils_tf.get_graph(graphs_tuple, index) graphs2 = utils_tf.nest_to_numpy(graphs2) actual = utils_np.graphs_tuple_to_data_dicts(graphs2) for ex, ac in zip(expected, actual): for k, v in ex.items(): self.assertAllClose(v, ac[k]) self.assertEqual(ex["nodes"].shape[0], ac["n_node"]) self.assertEqual(ex["edges"].shape[0], ac["n_edge"])
def test_getitem(self): index = slice(1, 3) expected = self.graphs_dicts_out[index] graphs_tuple = utils_tf.data_dicts_to_graphs_tuple( self.graphs_dicts_in) graphs2_op = utils_tf.get_graph(graphs_tuple, index) graphs2_op = utils_tf.make_runnable_in_session(graphs2_op) with self.test_session() as sess: graphs2 = sess.run(graphs2_op) actual = utils_np.graphs_tuple_to_data_dicts(graphs2) for ex, ac in zip(expected, actual): for k, v in ex.items(): self.assertAllClose(v, ac[k]) self.assertEqual(ex["nodes"].shape[0], ac["n_node"]) self.assertEqual(ex["edges"].shape[0], ac["n_edge"])
def _unpack_graphs_tuple(graphs_tuple, max_n_node, max_n_edge): """Unpacks data from a GraphTuple instance. Args: graphs_tuple: A GraphTuple instance. max_n_node: Maximum number of nodes. max_n_edge: Maximum number of edges. Returns: batch_nodes: A [batch, max_n_node, dims] float tensor. batch_edges: A [batch, max_n_edge, dims] float tensor. """ batch_nodes = [] batch_edges = [] batch_size = graphs_tuple.n_node.shape[0] for i in range(batch_size): graph_tuple_at_i = utils_tf.get_graph(graphs_tuple, i) n_node = tf.squeeze(graph_tuple_at_i.n_node) n_edge = tf.squeeze(graph_tuple_at_i.n_edge) nodes = graph_tuple_at_i.nodes edges = graph_tuple_at_i.edges if not None in [nodes, max_n_node]: batch_nodes.append( tf.pad(tensor=nodes, paddings=[[0, max_n_node - n_node], [0, 0]])) if not None in [edges, max_n_edge]: batch_edges.append( tf.pad(tensor=edges, paddings=[[0, max_n_edge - n_edge], [0, 0]])) batch_nodes = tf.stack(batch_nodes, 0) if batch_nodes else None batch_edges = tf.stack(batch_edges, 0) if batch_edges else None return batch_nodes, batch_edges
def test_raises(self, index, error_type, message): graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(self.graphs_dicts_in) with self.assertRaisesRegexp(error_type, message): utils_tf.get_graph(graphs_tuple, index)
x = tf.placeholder(shape=[None, features.shape[1]], dtype=tf.float32) y = tf.placeholder(shape=[None, y_train.shape[1]], dtype=tf.float32) train_mask_holder = tf.placeholder(shape=[ None, ], dtype=tf.int32) # define a Graph object input_graphs.append({"nodes": x}) input_graphs = utils_tf.data_dicts_to_graphs_tuple(input_graphs) input_graphs = utils_tf.fully_connect_graph_dynamic(input_graphs) adj = tf.constant(adj.toarray(), dtype=tf.float32) model = GCN() input_graph = utils_tf.get_graph(input_graphs, 0) # the output_graph = make_linear_model()(input_graph) # the output_graph.nodes = make_linear_model()(input_graph.nodes) output_graph = model(input_graph) # Make the graph can be ran in a session output_graph, input_graph = make_all_runnable_in_session( output_graph, input_graph) # Define the loss function loss = tf.nn.softmax_cross_entropy_with_logits( labels=tf.boolean_mask(y, train_mask_holder), logits=tf.boolean_mask(output_graph.nodes, train_mask_holder))
def _parse_single_example(example, options): """Parses a single tf.Example proto. Args: example: An Example proto. options: An instance of reader_pb2.Reader. Returns: A dictionary indexed by tensor name. """ # Initialize `keys_to_features`. example_fmt = { 'id': tf.io.FixedLenFeature([], tf.int64, default_value=-1), # Proposals 'image/n_proposal': tf.io.FixedLenFeature([], tf.int64, default_value=0), 'image/proposal/bbox/ymin': tf.io.VarLenFeature(tf.float32), 'image/proposal/bbox/xmin': tf.io.VarLenFeature(tf.float32), 'image/proposal/bbox/ymax': tf.io.VarLenFeature(tf.float32), 'image/proposal/bbox/xmax': tf.io.VarLenFeature(tf.float32), 'image/proposal/feature': tf.io.VarLenFeature(tf.float32), # Caption graph. 'caption_graph/caption': tf.io.VarLenFeature(tf.string), 'caption_graph/n_node': tf.io.VarLenFeature(tf.int64), 'caption_graph/n_edge': tf.io.VarLenFeature(tf.int64), 'caption_graph/nodes': tf.io.VarLenFeature(tf.string), 'caption_graph/edges': tf.io.VarLenFeature(tf.string), 'caption_graph/senders': tf.io.VarLenFeature(tf.int64), 'caption_graph/receivers': tf.io.VarLenFeature(tf.int64), # Ground-truth. 'scene_graph/n_relation': tf.io.FixedLenFeature([], tf.int64, default_value=0), 'scene_graph/predicate': tf.io.VarLenFeature(tf.string), 'scene_graph/subject': tf.io.VarLenFeature(tf.string), 'scene_graph/subject/bbox/ymin': tf.io.VarLenFeature(tf.float32), 'scene_graph/subject/bbox/xmin': tf.io.VarLenFeature(tf.float32), 'scene_graph/subject/bbox/ymax': tf.io.VarLenFeature(tf.float32), 'scene_graph/subject/bbox/xmax': tf.io.VarLenFeature(tf.float32), 'scene_graph/object': tf.io.VarLenFeature(tf.string), 'scene_graph/object/bbox/ymin': tf.io.VarLenFeature(tf.float32), 'scene_graph/object/bbox/xmin': tf.io.VarLenFeature(tf.float32), 'scene_graph/object/bbox/ymax': tf.io.VarLenFeature(tf.float32), 'scene_graph/object/bbox/xmax': tf.io.VarLenFeature(tf.float32), } # Decode proposals. parsed = tf.parse_single_example(example, example_fmt) proposals = tfexample_decoder.BoundingBox( prefix='image/proposal/bbox/').tensors_to_item(parsed) n_proposal = tf.minimum(parsed['image/n_proposal'], options.max_n_proposal) proposals = proposals[:options.max_n_proposal, :] proposal_features = tf.reshape( tf.sparse_tensor_to_dense(parsed['image/proposal/feature']), [-1, options.feature_dimensions])[:options.max_n_proposal, :] # Decode and randomly sample a caption graph. graphs = GraphsTuple( globals=None, n_node=tf.sparse_tensor_to_dense(parsed['caption_graph/n_node'], 0), n_edge=tf.sparse_tensor_to_dense(parsed['caption_graph/n_edge'], 0), nodes=tf.sparse_tensor_to_dense(parsed['caption_graph/nodes'], ''), edges=tf.sparse_tensor_to_dense(parsed['caption_graph/edges'], ''), senders=tf.sparse_tensor_to_dense(parsed['caption_graph/senders'], 0), receivers=tf.sparse_tensor_to_dense(parsed['caption_graph/receivers'], 0)) num_graphs = utils_tf.get_num_graphs(graphs) index = tf.random.uniform([], minval=0, maxval=num_graphs, dtype=tf.int32) caption = tf.sparse_tensor_to_dense(parsed['caption_graph/caption'])[index] caption_graph = utils_tf.get_graph(graphs, index) # Decode the ground-truth. subject_boxes = tfexample_decoder.BoundingBox( prefix='scene_graph/subject/bbox/').tensors_to_item(parsed) object_boxes = tfexample_decoder.BoundingBox( prefix='scene_graph/object/bbox/').tensors_to_item(parsed) feature_dict = { 'id': parsed['id'], # Proposals. 'image/n_proposal': n_proposal, 'image/proposal': proposals, 'image/proposal/feature': proposal_features, # Caption graph. 'caption_graph/caption': caption, 'caption_graph/n_node': caption_graph.n_node[0], 'caption_graph/n_edge': caption_graph.n_edge[0], 'caption_graph/nodes': caption_graph.nodes, 'caption_graph/edges': caption_graph.edges, 'caption_graph/senders': caption_graph.senders, 'caption_graph/receivers': caption_graph.receivers, # Ground-truth. 'scene_graph/n_relation': parsed['scene_graph/n_relation'], 'scene_graph/predicate': tf.sparse_tensor_to_dense(parsed['scene_graph/predicate'], ''), 'scene_graph/subject': tf.sparse_tensor_to_dense(parsed['scene_graph/subject'], ''), 'scene_graph/subject/box': subject_boxes, 'scene_graph/object': tf.sparse_tensor_to_dense(parsed['scene_graph/object'], ''), 'scene_graph/object/box': object_boxes, } for key in feature_dict.keys(): if key != 'id' and feature_dict[key].dtype == tf.int64: feature_dict[key] = tf.cast(feature_dict[key], tf.int32) return feature_dict
# define a Graph object input_graphs.append({"nodes": x, "edges": edges, "receivers": receivers, "senders": senders}) input_graphs = utils_tf.data_dicts_to_graphs_tuple(input_graphs) model = GCN() output_graphs = model(input_graphs) # Make the graph can be ran in a session # Define the loss function output_graph = utils_tf.get_graph(output_graphs, 0) output_graphs, input_graphs, output_graph = make_all_runnable_in_session( output_graphs, input_graphs, output_graph) loss = tf.nn.softmax_cross_entropy_with_logits( labels=tf.boolean_mask( y, train_mask_holder), logits=tf.boolean_mask( output_graph.nodes, train_mask_holder)) # define the optimizer optimizer = tf.train.AdamOptimizer(learning_rate=args.lr) shape = train_mask.shape[0] step_op = optimizer.minimize(loss) sess = tf.Session() # initialize all the variables in the graph. sess.run(tf.global_variables_initializer())