def create_data(batch_size, num_elements_min_max):
"""Returns graphs containing the inputs and targets for classification.
Refer to create_data_dicts_tf and create_linked_list_target for more details.
Args:
batch_size: batch size for the `input_graphs`.
num_elements_min_max: a 2-`tuple` of `int`s which define the [lower, upper)
range of the number of elements per list.
Returns:
inputs: a `graphs.GraphsTuple` which contains the input list as a graph.
targets: a `graphs.GraphsTuple` which contains the target as a graph.
sort_indices: a `graphs.GraphsTuple` which contains the sort indices of
the list elements a graph.
ranks: a `graphs.GraphsTuple` which contains the ranks of the list
elements as a graph.
"""
inputs, sort_indices, ranks = create_graph_dicts_tf(
batch_size, num_elements_min_max)
inputs = utils_tf.data_dicts_to_graphs_tuple(inputs)
sort_indices = utils_tf.data_dicts_to_graphs_tuple(sort_indices)
ranks = utils_tf.data_dicts_to_graphs_tuple(ranks)
inputs = utils_tf.fully_connect_graph_dynamic(inputs)
sort_indices = utils_tf.fully_connect_graph_dynamic(sort_indices)
ranks = utils_tf.fully_connect_graph_dynamic(ranks)
targets = create_linked_list_target(batch_size, sort_indices)
nodes = tf.concat((targets.nodes, 1.0 - targets.nodes), axis=1)
edges = tf.concat((targets.edges, 1.0 - targets.edges), axis=1)
targets = targets._replace(nodes=nodes, edges=edges)
return inputs, targets, sort_indices, ranks # input node[7,1] edge target edge[49,2] node[7,2]
def object_encoder_graph(self, encodings_per_object):
"""Function to make a graph object from the object oriented encoding
Inputs:
encodings_per_object: tensor containing the embeddings per object slot (size : [batch, n_objects, embedding dim])
Outputs:
graph_representation: fully conected graph with as node attributes the object encodings (size : [batch, graph])
"""
# Specify the number of nodes and edges implied by to the passed n_objects and the encodigs for each object
n_nodes = tf.tile(tf.constant([self.n_objects]),
tf.shape(encodings_per_object)[0:1],
name='n_nodes')
n_edges = tf.tile(tf.constant([0]),
tf.shape(encodings_per_object)[0:1],
name='n_edges')
# put the node_attributes in the correct shape to make graph object:
node_attributes = tf.reshape(encodings_per_object, [
tf.shape(encodings_per_object)[0] *
tf.shape(encodings_per_object)[1], self.state_dim_embedding
])
# make graph object with specified node attributes:
graph = graphs.GraphsTuple(nodes=node_attributes,
edges=None,
globals=None,
receivers=None,
senders=None,
n_node=n_nodes,
n_edge=n_edges)
# Connect all node to the other nodes (i.e. make fully connected):
fully_connected_graph = utils_tf.fully_connect_graph_dynamic(
graph, exclude_self_edges=False)
# Make it runnable in TF because None's are used:
runnable_fc_graph = utils_tf.make_runnable_in_session(
fully_connected_graph)
return runnable_fc_graph
def _build(self, batch, *args, **kwargs):
(graph, img, c) = batch
del c
# The encoded cluster graph has globals which can be compared against the encoded image graph
encoded_graph = self.epd_graph(graph, self._core_steps)
# Add an extra dimension to the image (tf.summary expects a Tensor of rank 4)
img = img[None, ...]
im_before_cnn = (img - tf.reduce_min(img)) / (tf.reduce_max(img) - tf.reduce_min(img))
tf.summary.image(f'img_before_cnn', im_before_cnn, step=self.step)
img = self.auto_encoder.encoder(img)
# Prevent the autoencoder from learning
try:
for variable in self.auto_encoder.encoder.trainable_variables:
variable._trainable = False
for variable in self.auto_encoder.decoder.trainable_variables:
variable._trainable = False
except:
pass
img_after_autoencoder = (img - tf.reduce_min(img)) / (tf.reduce_max(img) - tf.reduce_min(img))
tf.summary.image(f'img_after_autoencoder', tf.transpose(img_after_autoencoder, [3, 1, 2, 0]), step=self.step)
decoded_img = self.auto_encoder.decoder(img)
decoded_img = (decoded_img - tf.reduce_min(decoded_img)) / (tf.reduce_max(decoded_img) - tf.reduce_min(decoded_img))
tf.summary.image(f'decoded_img', decoded_img, step=self.step)
# Reshape the encoded image so it can be used for the nodes
#1, w,h,c -> w*h, c
nodes = tf.reshape(img, (-1,self.image_feature_size))
# Create a graph that has a node for every encoded pixel. The features of each node
# are the channels of the corresponding pixel. Then connect each node with every other
# node.
img_graph = GraphsTuple(nodes=nodes,
edges=None,
globals=None,
receivers=None,
senders=None,
n_node=tf.shape(nodes)[0:1],
n_edge=tf.constant([0]))
connected_graph = fully_connect_graph_dynamic(img_graph)
# The encoded image graph has globals which can be compared against the encoded cluster graph
encoded_img = self.epd_image(connected_graph, 1)
# Compare the globals from the encoded cluster graph and encoded image graph
# to estimate the similarity between the input graph and input image
distance = self.compare(tf.concat([encoded_graph.globals, encoded_img.globals], axis=1)) + self.compare(
tf.concat([encoded_img.globals, encoded_graph.globals], axis=1))
return distance
def get_batched_graphs (train_set):
"""
description: converts inputs in each batch to complete graphs
:param train_set: training set containing tuples (batch_input , batch_target)
:return:
"""
for batch_input , batch_target in train_set:
input_dict = create_graph_dicts(batch_input)
targets = batch_target
input_dict = utils_tf.data_dicts_to_graphs_tuple(input_dict)
input_dict = utils_tf.fully_connect_graph_dynamic(input_dict)
yield input_dict , targets
def sample_decoder(self, positions, logits, temperature):
token_distribution = tfp.distributions.RelaxedOneHotCategorical(temperature, logits=logits)
token_samples_onehot = token_distribution.sample((1,),
name='token_samples')
token_sample_onehot = token_samples_onehot[0] # [n_node, num_embedding]
token_sample = tf.matmul(token_sample_onehot, self.embeddings) # [n_node, embedding_dim]
n_node = tf.shape(token_sample)[0]
latent_graph = GraphsTuple(nodes=token_sample,
edges=None,
globals=tf.constant([0.], dtype=tf.float32),
senders=None,
receivers=None,
n_node=tf.constant([n_node], dtype=tf.int32),
n_edge=tf.constant([0], dtype=tf.int32)) # [n_node, embedding_dim]
latent_graph = fully_connect_graph_dynamic(latent_graph)
gaussian_tokens = self.decoder(latent_graph) # nodes=[num_gaussian_components, component_dim]
reconstructed_fields = reconstruct_fields_from_gaussians(gaussian_tokens, positions)
return reconstructed_fields
def _single_decode(token_sample_onehot):
"""
Args:
token_sample: [n_node, embedding_dim]
Returns:
log_likelihood: scalar
kl_term: scalar
"""
token_sample = tf.matmul(
token_sample_onehot,
self.embeddings) # [n_node, embedding_dim] # = z ~ q(z|x)
latent_graph = GraphsTuple(
nodes=token_sample,
edges=None,
globals=tf.constant([0.], dtype=tf.float32),
senders=None,
receivers=None,
n_node=encoded_graph.n_node,
n_edge=tf.constant([0],
dtype=tf.int32)) # [n_node, embedding_dim]
latent_graph = fully_connect_graph_dynamic(latent_graph)
# print('\n latent_graph', latent_graph, '\n')
gaussian_tokens = self.decoder(
latent_graph) # nodes=[num_gaussian_components, component_dim]
# print('\n gaussian_tokens_nodes', gaussian_tokens.nodes, '\n')
_, log_likelihood = gaussian_loss_function(gaussian_tokens.nodes,
graph)
# [n_node, num_embeddings].[n_node, num_embeddings]
sum_selected_logits = tf.math.reduce_sum(token_sample_onehot *
logits,
axis=1) # [n_node]
# print('sum', sum_selected_logits)
# print('norm', log_norm)
# print('num_embed', tf.cast(self.num_embedding, tf.float32))
# print('embed', tf.math.log(tf.cast(self.num_embedding, tf.float32)))
kl_term = sum_selected_logits - self.num_embedding * log_norm + \
self.num_embedding * tf.math.log(tf.cast(self.num_embedding, tf.float32))
# print('kl_term 0', kl_term)
# print('kl_term', tf.reduce_mean(kl_term))
kl_term = self.beta * tf.reduce_mean(kl_term)
return log_likelihood, kl_term
def test_fully_connect_graph_dynamic_with_dynamic_sizes(
self, exclude_self_edges):
for g in self.graphs_dicts_in:
g.pop("edges")
g.pop("receivers")
g.pop("senders")
n_relation = 0
for g in self.graphs_dicts_in:
n_node = g["nodes"].shape[0]
if exclude_self_edges:
n_relation += n_node * (n_node - 1)
else:
n_relation += n_node * n_node
graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(
self.graphs_dicts_in)
graphs_tuple = graphs_tuple.map(
test_utils.mask_leading_dimension,
["nodes", "globals", "n_node", "n_edge"])
graphs_tuple = utils_tf.fully_connect_graph_dynamic(
graphs_tuple, exclude_self_edges)
with self.test_session() as sess:
actual_receivers, actual_senders, actual_n_edge = sess.run([
graphs_tuple.receivers, graphs_tuple.senders,
graphs_tuple.n_edge
])
self.assertAllEqual((n_relation, ), actual_receivers.shape)
self.assertAllEqual((n_relation, ), actual_senders.shape)
self.assertAllEqual((len(self.graphs_dicts_in), ), actual_n_edge.shape)
expected_edges = []
offset = 0
for graph in self.graphs_dicts_in:
n_node = graph["nodes"].shape[0]
for e1 in range(n_node):
for e2 in range(n_node):
if not exclude_self_edges or e1 != e2:
expected_edges.append((e1 + offset, e2 + offset))
offset += n_node
actual_edges = zip(actual_receivers, actual_senders)
self.assertSetEqual(set(actual_edges), set(expected_edges))
def test_fully_connect_graph_dynamic(self, exclude_self_edges):
for g in self.graphs_dicts_in:
g.pop("edges")
g.pop("receivers")
g.pop("senders")
n_relation = 0
for g in self.graphs_dicts_in:
n_node = g["nodes"].shape[0]
if exclude_self_edges:
n_relation += n_node * (n_node - 1)
else:
n_relation += n_node * n_node
graphs_tuple = utils_tf.data_dicts_to_graphs_tuple(self.graphs_dicts_in)
graphs_tuple = utils_tf.fully_connect_graph_dynamic(graphs_tuple,
exclude_self_edges)
with self.test_session() as sess:
actual_receivers, actual_senders = sess.run(
[graphs_tuple.receivers, graphs_tuple.senders])
self.assertAllEqual((n_relation,), actual_receivers.shape)
self.assertAllEqual((n_relation,), actual_senders.shape)
self.assertAllEqual((len(self.graphs_dicts_in),),
graphs_tuple.n_edge.get_shape().as_list())
def _build(self, batch, *args, **kwargs):
(graph, img, c) = batch
del c
encoded_graph = self.encoder_graph(graph)
tf.summary.image(f'img_before_cnn', img[None, ...], step=self.step)
img = self.image_cnn(img[None, ...])
for channel in range(img.shape[-1]):
tf.summary.image(f'img_after_cnn[{channel}]',
img[..., channel:channel + 1],
step=self.step)
#1, w,h,c -> w*h, c
nodes = tf.reshape(img, (-1, self.image_feature_size))
img_graph = GraphsTuple(nodes=nodes,
edges=None,
globals=None,
receivers=None,
senders=None,
n_node=tf.shape(nodes)[0:1],
n_edge=tf.constant([0]))
connected_graph = fully_connect_graph_dynamic(img_graph)
encoded_img = self.encoder_image(connected_graph)
return self.compare(
tf.concat([encoded_graph.globals, encoded_img.globals],
axis=1)) #[1]
def _build(self, graph):
# give graph edges and new node dimension (linear transformation)
graph = graph.replace(edges=tf.tile(self.intra_graph_edge_variable[None, :], [graph.n_edge[0], 1]))
graph = self.projection_node_block(graph) # [n_nodes, node_size]
# print('graph 1', graph)
n_node = tf.shape(graph.nodes)[0]
graph.replace(n_node=n_node)
# create fully connected output token nodes
token_start_nodes = tf.tile(self.empty_node_variable[None, :], [self.num_output, 1])
token_graph = GraphsTuple(nodes=token_start_nodes,
edges=None,
globals=tf.constant([0.], dtype=tf.float32),
senders=None,
receivers=None,
n_node=tf.constant([self.num_output], dtype=tf.int32),
n_edge=tf.constant([0], dtype=tf.int32))
token_graph = fully_connect_graph_dynamic(token_graph)
# print('\n token graph', token_graph, '\n')
token_graph = token_graph.replace(
edges=tf.tile(self.intra_token_graph_edge_variable[None, :], [token_graph.n_edge[0], 1]))
concat_graph = concat([graph, token_graph], axis=0) # n_node = [n_nodes, n_tokes]
concat_graph = concat_graph.replace(n_node=tf.reduce_sum(concat_graph.n_node, keepdims=True),
n_edge=tf.reduce_sum(concat_graph.n_edge,
keepdims=True)) # n_node=[n_nodes+n_tokens]
# add random edges between
# choose random unique set of nodes in graph, choose random set of nodes in token_graph
gumbel = -tf.math.log(-tf.math.log(tf.random.uniform((n_node,))))
n_connect_edges = tf.cast(
tf.multiply(tf.constant([self.inter_graph_connect_prob]), tf.cast(n_node, tf.float32)), tf.int32)
_, graph_senders = tf.nn.top_k(gumbel, n_connect_edges[0])
# print('graph_senders', graph_senders)
token_graph_receivers = n_node + tf.random.uniform(shape=n_connect_edges, minval=0, maxval=self.num_output,
dtype=tf.int32)
# print('token_graph_receivers', token_graph_receivers)
senders = tf.concat([concat_graph.senders, graph_senders, token_graph_receivers],
axis=0) # add bi-directional senders + receivers
receivers = tf.concat([concat_graph.receivers, token_graph_receivers, graph_senders], axis=0)
inter_edges = tf.tile(self.inter_graph_edge_variable[None, :],
tf.concat([2 * n_connect_edges, tf.constant([1], dtype=tf.int32)],
axis=0)) # 200 = 10000(n_nodes) * 0.01 * 2
edges = tf.concat([concat_graph.edges, inter_edges], axis=0)
concat_graph = concat_graph.replace(senders=senders, receivers=receivers, edges=edges,
n_edge=concat_graph.n_edge[0] + 2 * n_connect_edges[0],
# concat_graph.n_edge[0] + 2 * n_connect_edges
globals=self.starting_global_variable[None, :])
# print('starting global', self.starting_global_variable[None, :])
latent_graph = concat_graph
print('concat_graph_nodes', self.name, concat_graph.nodes)
for step in range(self.crossing_steps): # this would be that theoretical crossing time for information through the graph
input_nodes = latent_graph.nodes
latent_graph = self.edge_block(latent_graph)
latent_graph = self.node_block(latent_graph)
latent_graph = self.global_block(latent_graph)
latent_graph = latent_graph.replace(nodes=latent_graph.nodes + input_nodes) # residual connections
print('latent_graph_nodes', self.name, latent_graph.nodes)
print('latent_graph_edges', self.name, latent_graph.edges)
print('latent_graph_globals', self.name, latent_graph.globals)
latent_graph = latent_graph.replace(nodes=latent_graph.nodes[n_node:],
edges=None,
receivers=None,
senders=None,
globals=None,
n_node=tf.constant([self.num_output], dtype=tf.int32),
n_edge=tf.constant(0, dtype=tf.int32))
output_graph = self.output_projection_node_block(latent_graph)
print('output_graph_nodes', self.name, output_graph.nodes)
return output_graph
def make_all_runnable_in_session(*args):
"""Lets an iterable of TF graphs be output from a session as NP graphs."""
return [utils_tf.make_runnable_in_session(a) for a in args]
x = tf.placeholder(shape=[None, features.shape[1]], dtype=tf.float32)
y = tf.placeholder(shape=[None, y_train.shape[1]], dtype=tf.float32)
train_mask_holder = tf.placeholder(shape=[
None,
], dtype=tf.int32)
# define a Graph object
input_graphs.append({"nodes": x})
input_graphs = utils_tf.data_dicts_to_graphs_tuple(input_graphs)
input_graphs = utils_tf.fully_connect_graph_dynamic(input_graphs)
adj = tf.constant(adj.toarray(), dtype=tf.float32)
model = GCN()
input_graph = utils_tf.get_graph(input_graphs, 0)
# the output_graph = make_linear_model()(input_graph)
# the output_graph.nodes = make_linear_model()(input_graph.nodes)
output_graph = model(input_graph)
# Make the graph can be ran in a session
output_graph, input_graph = make_all_runnable_in_session(
output_graph, input_graph)
# Define the loss function
loss = tf.nn.softmax_cross_entropy_with_logits(
def _build(self, batch, *args, **kwargs):
(graph, img, c) = batch
del c
print(graph.nodes.shape)
# The encoded cluster graph has globals which can be compared against the encoded image graph
encoded_graph = self.epd_graph(graph, self._core_steps)
# # Add an extra dimension to the image (tf.summary expects a Tensor of rank 4)
# img = img[None, ...]
print("IMG SHAPE:", img.shape)
print("IMG MIN MAX:", tf.math.reduce_min(img), tf.math.reduce_max(img))
img_before_cnn = (img - tf.reduce_min(img)) / \
(tf.reduce_max(img) - tf.reduce_min(img))
tf.summary.image(f'img_before_cnn', img_before_cnn, step=self.step)
# Smooth the image and use the encoder from the autoencoder to reduce the dimensionality of the image
# The autoencoder was trained on images that were smoothed in the same way
img = gaussian_filter2d(img, filter_shape=[6, 6])
img = self.auto_encoder.encoder(img)
# Prevent the autoencoder from learning
try:
for variable in self.auto_encoder.encoder.trainable_variables:
variable._trainable = False
for variable in self.auto_encoder.decoder.trainable_variables:
variable._trainable = False
except:
pass
print("IMG SHAPE AFTER CNN:", img.shape)
print("IMG MIN MAX AFTER CNN:", tf.math.reduce_min(img),
tf.math.reduce_max(img))
img_after_cnn = (img - tf.reduce_min(img)) / \
(tf.reduce_max(img) - tf.reduce_min(img))
tf.summary.image(
f'quantized_img',
img_after_cnn[:, :, :, np.random.randint(low=0, high=64)][:, :, :,
None],
step=self.step)
decoded_img = self.auto_encoder.decoder(img)
decoded_img = (decoded_img - tf.reduce_min(decoded_img)) / \
(tf.reduce_max(decoded_img) - tf.reduce_min(decoded_img))
tf.summary.image(f'decoded_img', decoded_img, step=self.step)
# Reshape the encoded image so it can be used for the nodes
img_nodes = tf.reshape(img, (-1, self.image_feature_size))
print(img_nodes.shape)
# Create a graph that has a node for every encoded pixel. The features of each node
# are the channels of the corresponding pixel. Then connect each node with every other
# node.
img_graph = GraphsTuple(nodes=img_nodes,
edges=None,
globals=None,
receivers=None,
senders=None,
n_node=tf.shape(img_nodes)[0:1],
n_edge=tf.constant([0]))
connected_graph = fully_connect_graph_dynamic(img_graph)
# The encoded image graph has globals which can be compared against the encoded cluster graph
encoded_img = self.epd_image(connected_graph, 1)
# Compare the globals from the encoded cluster graph and encoded image graph
# to estimate the similarity between the input graph and input image
print(encoded_img.globals.shape)
print(encoded_graph.globals.shape)
distance = self.compare(
tf.concat([encoded_graph.globals, encoded_img.globals], axis=1)
) + self.compare(
tf.concat([encoded_img.globals, encoded_graph.globals], axis=1))
return distance
