def compute_dist_bacc(predicted, ground_truth):
n_graphs = predicted.n_node.shape[0]
accs = np.zeros(n_graphs)
for idx in range(n_graphs):
pred_graph = utils_np.get_graph(predicted, idx)
gt_graph = utils_np.get_graph(ground_truth, idx)
acc = get_accuracy(pred_graph.edges, gt_graph.edges, bidim=False)
accs[idx] = acc
return accs
def test_get_single_item(self):
index = 2
expected = self.graphs_dicts_out[index]
graphs = utils_np.data_dicts_to_graphs_tuple(self.graphs_dicts_in)
graph = utils_np.get_graph(graphs, index)
actual, = utils_np.graphs_tuple_to_data_dicts(graph)
for k, v in expected.items():
self.assertAllClose(v, actual[k])
def test_get_many_items(self):
index = slice(1, 3)
expected = self.graphs_dicts_out[index]
graphs = utils_np.data_dicts_to_graphs_tuple(self.graphs_dicts_in)
graphs2 = utils_np.get_graph(graphs, index)
actual = utils_np.graphs_tuple_to_data_dicts(graphs2)
for ex, ac in zip(expected, actual):
for k, v in ex.items():
self.assertAllClose(v, ac[k])
def save_examples(generator, save_dir=None,
examples_per_file=26, num_examples=1, prefix='train'):
"""
Saves a list of GraphTuples to tfrecords.
Args:
generator: generator (or list) of (GraphTuples, image).
Generator is more efficient.
save_dir: dir to save tfrecords in
examples_per_file: int, max number examples per file
Returns: list of tfrecord files.
"""
print("Saving data in tfrecords.")
if save_dir is None:
save_dir = os.getcwd()
os.makedirs(save_dir, exist_ok=True)
files = []
data_iterable = iter(generator)
data_left = True
pbar = tqdm(total=num_examples)
while data_left:
mp_lock.acquire() # make sure no duplicate files are made / replaced
tf_files = glob.glob(os.path.join(save_dir, 'train_*'))
file_idx = len(tf_files)
mp_lock.release()
file = os.path.join(save_dir, 'train_{:04d}.tfrecords'.format(file_idx))
files.append(file)
with tf.io.TFRecordWriter(file) as writer:
for i in range(examples_per_file):
try:
(graph, image, example_idx) = next(data_iterable)
except StopIteration:
data_left = False
break
graph = get_graph(graph, 0)
features = dict(
image=tf.train.Feature(
bytes_list=tf.train.BytesList(
value=[tf.io.serialize_tensor(tf.cast(image, tf.float32)).numpy()])),
example_idx=tf.train.Feature(
bytes_list=tf.train.BytesList(
value=[tf.io.serialize_tensor(tf.cast(example_idx, tf.int32)).numpy()])),
**graph_tuple_to_feature(graph, name='graph')
)
features = tf.train.Features(feature=features)
example = tf.train.Example(features=features)
writer.write(example.SerializeToString())
pbar.update(1)
print("Saved in tfrecords: {}".format(files))
return files
def get_stages(in_graphs, gt_graphs, pred_graphs, bidim=True):
all_stages = dict(transient={
"cost": [],
"hops": []
},
steady={
"cost": [],
"hops": []
})
n_graphs = len(in_graphs.n_node)
bar = tqdm(total=n_graphs, desc="Processed Graphs")
for graph_idx in range(n_graphs):
pred_graph = utils_np.get_graph(pred_graphs, graph_idx)
if bidim:
pred_graph = parse_edges_bidim_probs(pred_graph)
gt_graph = utils_np.get_graph(gt_graphs, graph_idx)
end_node = np.argwhere(gt_graph.nodes[:, 0] == 0).reshape(1)[0]
djk_cost = gt_graph.nodes[:, 0]
djk_hops = gt_graph.nodes[:, 1]
edge_weights = utils_np.get_graph(in_graphs, graph_idx).edges
stages = reverse_link(pred_graph,
end_node,
djk_cost,
djk_hops,
edge_weights,
bidim=bidim)
all_stages["transient"]["cost"] += stages["transient"]["cost"]
all_stages["transient"]["hops"] += stages["transient"]["hops"]
all_stages["steady"]["cost"] += stages["steady"]["cost"]
all_stages["steady"]["hops"] += stages["steady"]["hops"]
bar.update()
bar.set_postfix(cost_transient=np.array(
stages["transient"]["cost"]).mean(),
cost_steady=np.array(stages["steady"]["cost"]).mean())
bar.close()
all_stages["transient"]["cost"] = np.array(all_stages["transient"]["cost"])
all_stages["transient"]["hops"] = np.array(all_stages["transient"]["hops"])
all_stages["steady"]["cost"] = np.array(all_stages["steady"]["cost"])
all_stages["steady"]["hops"] = np.array(all_stages["steady"]["hops"])
return all_stages
def save_examples(generator, save_dir=None, num_examples=1):
"""
Saves a list of GraphTuples to tfrecords.
Args:
generator: generator (or list) of (GraphTuples, image).
Generator is more efficient.
save_dir: dir to save tfrecords in
examples_per_file: int, max number examples per file
Returns: list of tfrecord files.
"""
print("Saving data in tfrecords.")
if save_dir is None:
save_dir = os.getcwd()
os.makedirs(save_dir, exist_ok=True)
data_iterable = iter(generator)
pbar = tqdm(total=num_examples)
file = os.path.join(save_dir, 'examples.tfrecords')
with tf.io.TFRecordWriter(file) as writer:
for (graph, image, example_idx) in data_iterable:
graph = get_graph(graph, 0)
features = dict(
image=tf.train.Feature(bytes_list=tf.train.BytesList(value=[
tf.io.serialize_tensor(tf.cast(image, tf.float32)).numpy()
])),
example_idx=tf.train.Feature(bytes_list=tf.train.BytesList(
value=[
tf.io.serialize_tensor(tf.cast(example_idx,
tf.int32)).numpy()
])),
**graph_tuple_to_feature(graph, name='graph'))
features = tf.train.Features(feature=features)
example = tf.train.Example(features=features)
writer.write(example.SerializeToString())
pbar.update(1)
files = [file]
print("Saved in tfrecords: {}".format(files))
return files
def save_examples(generator,
save_dir=None,
examples_per_file=32,
num_examples=1,
exp_time=None,
prefix='train'):
"""
Saves a list of GraphTuples to tfrecords.
Args:
generator: generator (or list) of (GraphTuples, image).
Generator is more efficient.
save_dir: dir to save tfrecords in
examples_per_file: int, max number examples per file
num_examples: number of examples
exp_time: exposure time (used in filename)
Returns: list of tfrecord files.
"""
print("Saving data in tfrecords.")
# If the directory where to save the tfrecords is not specified, save them in the current working directory
if save_dir is None:
save_dir = os.getcwd()
# If the save directory does not yet exist, create it
os.makedirs(save_dir, exist_ok=True)
# Files will be returned
files = []
# next(data_iterable) gives the next dataset in the iterable
data_iterable = iter(generator)
data_left = True
# Status bar
pbar = tqdm(total=num_examples)
while data_left:
# For every 'examples_per_file' (=32) example directories, create a tf_records file
if exp_time is not None:
exp_time_str = f'_{int(exp_time)}ks_'
else:
exp_time_str = ''
mp_lock.acquire() # make sure no duplicate files are made / replaced
tf_files = glob.glob(os.path.join(save_dir, 'train_*'))
file_idx = len(tf_files)
indices = sorted(
[int(tf_file.split('.')[0][-4:]) for tf_file in tf_files])
for idx, ind in enumerate(indices):
if idx != ind:
file_idx = idx
break
file = os.path.join(
save_dir,
prefix + exp_time_str + '{:04d}.tfrecords'.format(file_idx))
files.append(file)
mp_lock.release()
# 'writer' can write to 'file'
with tf.io.TFRecordWriter(file) as writer:
for i in range(examples_per_file + 1):
# Yield a dataset extracted by the generator
try:
(graph, image, cluster_idx, projection_idx,
vprime) = next(data_iterable)
except StopIteration:
data_left = False
break
# Write the graph, image and example_idx to the tfrecord file
graph = get_graph(graph, 0)
features = dict(
image=tf.train.Feature(bytes_list=tf.train.BytesList(
value=[
tf.io.serialize_tensor(tf.cast(
image, tf.float32)).numpy()
])),
vprime=tf.train.Feature(bytes_list=tf.train.BytesList(
value=[
tf.io.serialize_tensor(tf.cast(
vprime, tf.float32)).numpy()
])),
cluster_idx=tf.train.Feature(bytes_list=tf.train.BytesList(
value=[
tf.io.serialize_tensor(
tf.cast(cluster_idx, tf.int32)).numpy()
])),
projection_idx=tf.train.Feature(
bytes_list=tf.train.BytesList(value=[
tf.io.serialize_tensor(
tf.cast(projection_idx, tf.int32)).numpy()
])),
**graph_tuple_to_feature(graph, name='graph'))
# Set the features up so they can be written to the tfrecord file
features = tf.train.Features(feature=features)
example = tf.train.Example(features=features)
writer.write(example.SerializeToString())
# Status bar update
pbar.update(1)
print("Saved in tfrecords: {}".format(files))
return files
# first_six_graphs_pred_np = utils_np.get_graph(test2, slice(5, 10))
# plot_graphs_tuple_np(first_six_graphs_pred_np,2)
# plt.show()
# test = utils_np.data_dicts_to_graphs_tuple(visualization_true_nodes_rollout_9_ge[example_number])
# first_six_graphs_np = utils_np.get_graph(test, slice(0, 5))
# plot_graphs_tuple_np(first_six_graphs_np,1)
# first_six_graphs_np = utils_np.get_graph(test, slice(5, 10))
# plot_graphs_tuple_np(first_six_graphs_np,2)
# plt.show()
for example_number in range(0, 5): # first 10 examples
test = utils_np.data_dicts_to_graphs_tuple(
visualization_true_nodes_rollout_9_ge[example_number])
test2 = utils_np.data_dicts_to_graphs_tuple(
prediction_over_time[example_number])
first_six_graphs_pred_np = utils_np.get_graph(test, slice(0, 10))
plot_graphs_tuple_np(first_six_graphs_pred_np, 1)
first_six_graphs_pred_np = utils_np.get_graph(test2, slice(0, 10))
plot_graphs_tuple_np(first_six_graphs_pred_np, 2)
plt.show()
for example_number in range(40, 50): # first 10 examples
test = utils_np.data_dicts_to_graphs_tuple(
visualization_true_nodes_rollout_9_ge[example_number])
test2 = utils_np.data_dicts_to_graphs_tuple(
prediction_over_time[example_number])
first_six_graphs_pred_np = utils_np.get_graph(test, slice(0, 10))
plot_graphs_tuple_np(first_six_graphs_pred_np, 1)
first_six_graphs_pred_np = utils_np.get_graph(test2, slice(0, 10))
plot_graphs_tuple_np(first_six_graphs_pred_np, 2)
plt.show()
