def compute_dist_bacc(predicted, ground_truth): n_graphs = predicted.n_node.shape[0] accs = np.zeros(n_graphs) for idx in range(n_graphs): pred_graph = utils_np.get_graph(predicted, idx) gt_graph = utils_np.get_graph(ground_truth, idx) acc = get_accuracy(pred_graph.edges, gt_graph.edges, bidim=False) accs[idx] = acc return accs
def test_get_single_item(self): index = 2 expected = self.graphs_dicts_out[index] graphs = utils_np.data_dicts_to_graphs_tuple(self.graphs_dicts_in) graph = utils_np.get_graph(graphs, index) actual, = utils_np.graphs_tuple_to_data_dicts(graph) for k, v in expected.items(): self.assertAllClose(v, actual[k])
def test_get_many_items(self): index = slice(1, 3) expected = self.graphs_dicts_out[index] graphs = utils_np.data_dicts_to_graphs_tuple(self.graphs_dicts_in) graphs2 = utils_np.get_graph(graphs, index) actual = utils_np.graphs_tuple_to_data_dicts(graphs2) for ex, ac in zip(expected, actual): for k, v in ex.items(): self.assertAllClose(v, ac[k])
def save_examples(generator, save_dir=None, examples_per_file=26, num_examples=1, prefix='train'): """ Saves a list of GraphTuples to tfrecords. Args: generator: generator (or list) of (GraphTuples, image). Generator is more efficient. save_dir: dir to save tfrecords in examples_per_file: int, max number examples per file Returns: list of tfrecord files. """ print("Saving data in tfrecords.") if save_dir is None: save_dir = os.getcwd() os.makedirs(save_dir, exist_ok=True) files = [] data_iterable = iter(generator) data_left = True pbar = tqdm(total=num_examples) while data_left: mp_lock.acquire() # make sure no duplicate files are made / replaced tf_files = glob.glob(os.path.join(save_dir, 'train_*')) file_idx = len(tf_files) mp_lock.release() file = os.path.join(save_dir, 'train_{:04d}.tfrecords'.format(file_idx)) files.append(file) with tf.io.TFRecordWriter(file) as writer: for i in range(examples_per_file): try: (graph, image, example_idx) = next(data_iterable) except StopIteration: data_left = False break graph = get_graph(graph, 0) features = dict( image=tf.train.Feature( bytes_list=tf.train.BytesList( value=[tf.io.serialize_tensor(tf.cast(image, tf.float32)).numpy()])), example_idx=tf.train.Feature( bytes_list=tf.train.BytesList( value=[tf.io.serialize_tensor(tf.cast(example_idx, tf.int32)).numpy()])), **graph_tuple_to_feature(graph, name='graph') ) features = tf.train.Features(feature=features) example = tf.train.Example(features=features) writer.write(example.SerializeToString()) pbar.update(1) print("Saved in tfrecords: {}".format(files)) return files
def get_stages(in_graphs, gt_graphs, pred_graphs, bidim=True): all_stages = dict(transient={ "cost": [], "hops": [] }, steady={ "cost": [], "hops": [] }) n_graphs = len(in_graphs.n_node) bar = tqdm(total=n_graphs, desc="Processed Graphs") for graph_idx in range(n_graphs): pred_graph = utils_np.get_graph(pred_graphs, graph_idx) if bidim: pred_graph = parse_edges_bidim_probs(pred_graph) gt_graph = utils_np.get_graph(gt_graphs, graph_idx) end_node = np.argwhere(gt_graph.nodes[:, 0] == 0).reshape(1)[0] djk_cost = gt_graph.nodes[:, 0] djk_hops = gt_graph.nodes[:, 1] edge_weights = utils_np.get_graph(in_graphs, graph_idx).edges stages = reverse_link(pred_graph, end_node, djk_cost, djk_hops, edge_weights, bidim=bidim) all_stages["transient"]["cost"] += stages["transient"]["cost"] all_stages["transient"]["hops"] += stages["transient"]["hops"] all_stages["steady"]["cost"] += stages["steady"]["cost"] all_stages["steady"]["hops"] += stages["steady"]["hops"] bar.update() bar.set_postfix(cost_transient=np.array( stages["transient"]["cost"]).mean(), cost_steady=np.array(stages["steady"]["cost"]).mean()) bar.close() all_stages["transient"]["cost"] = np.array(all_stages["transient"]["cost"]) all_stages["transient"]["hops"] = np.array(all_stages["transient"]["hops"]) all_stages["steady"]["cost"] = np.array(all_stages["steady"]["cost"]) all_stages["steady"]["hops"] = np.array(all_stages["steady"]["hops"]) return all_stages
def save_examples(generator, save_dir=None, num_examples=1): """ Saves a list of GraphTuples to tfrecords. Args: generator: generator (or list) of (GraphTuples, image). Generator is more efficient. save_dir: dir to save tfrecords in examples_per_file: int, max number examples per file Returns: list of tfrecord files. """ print("Saving data in tfrecords.") if save_dir is None: save_dir = os.getcwd() os.makedirs(save_dir, exist_ok=True) data_iterable = iter(generator) pbar = tqdm(total=num_examples) file = os.path.join(save_dir, 'examples.tfrecords') with tf.io.TFRecordWriter(file) as writer: for (graph, image, example_idx) in data_iterable: graph = get_graph(graph, 0) features = dict( image=tf.train.Feature(bytes_list=tf.train.BytesList(value=[ tf.io.serialize_tensor(tf.cast(image, tf.float32)).numpy() ])), example_idx=tf.train.Feature(bytes_list=tf.train.BytesList( value=[ tf.io.serialize_tensor(tf.cast(example_idx, tf.int32)).numpy() ])), **graph_tuple_to_feature(graph, name='graph')) features = tf.train.Features(feature=features) example = tf.train.Example(features=features) writer.write(example.SerializeToString()) pbar.update(1) files = [file] print("Saved in tfrecords: {}".format(files)) return files
def save_examples(generator, save_dir=None, examples_per_file=32, num_examples=1, exp_time=None, prefix='train'): """ Saves a list of GraphTuples to tfrecords. Args: generator: generator (or list) of (GraphTuples, image). Generator is more efficient. save_dir: dir to save tfrecords in examples_per_file: int, max number examples per file num_examples: number of examples exp_time: exposure time (used in filename) Returns: list of tfrecord files. """ print("Saving data in tfrecords.") # If the directory where to save the tfrecords is not specified, save them in the current working directory if save_dir is None: save_dir = os.getcwd() # If the save directory does not yet exist, create it os.makedirs(save_dir, exist_ok=True) # Files will be returned files = [] # next(data_iterable) gives the next dataset in the iterable data_iterable = iter(generator) data_left = True # Status bar pbar = tqdm(total=num_examples) while data_left: # For every 'examples_per_file' (=32) example directories, create a tf_records file if exp_time is not None: exp_time_str = f'_{int(exp_time)}ks_' else: exp_time_str = '' mp_lock.acquire() # make sure no duplicate files are made / replaced tf_files = glob.glob(os.path.join(save_dir, 'train_*')) file_idx = len(tf_files) indices = sorted( [int(tf_file.split('.')[0][-4:]) for tf_file in tf_files]) for idx, ind in enumerate(indices): if idx != ind: file_idx = idx break file = os.path.join( save_dir, prefix + exp_time_str + '{:04d}.tfrecords'.format(file_idx)) files.append(file) mp_lock.release() # 'writer' can write to 'file' with tf.io.TFRecordWriter(file) as writer: for i in range(examples_per_file + 1): # Yield a dataset extracted by the generator try: (graph, image, cluster_idx, projection_idx, vprime) = next(data_iterable) except StopIteration: data_left = False break # Write the graph, image and example_idx to the tfrecord file graph = get_graph(graph, 0) features = dict( image=tf.train.Feature(bytes_list=tf.train.BytesList( value=[ tf.io.serialize_tensor(tf.cast( image, tf.float32)).numpy() ])), vprime=tf.train.Feature(bytes_list=tf.train.BytesList( value=[ tf.io.serialize_tensor(tf.cast( vprime, tf.float32)).numpy() ])), cluster_idx=tf.train.Feature(bytes_list=tf.train.BytesList( value=[ tf.io.serialize_tensor( tf.cast(cluster_idx, tf.int32)).numpy() ])), projection_idx=tf.train.Feature( bytes_list=tf.train.BytesList(value=[ tf.io.serialize_tensor( tf.cast(projection_idx, tf.int32)).numpy() ])), **graph_tuple_to_feature(graph, name='graph')) # Set the features up so they can be written to the tfrecord file features = tf.train.Features(feature=features) example = tf.train.Example(features=features) writer.write(example.SerializeToString()) # Status bar update pbar.update(1) print("Saved in tfrecords: {}".format(files)) return files
# first_six_graphs_pred_np = utils_np.get_graph(test2, slice(5, 10)) # plot_graphs_tuple_np(first_six_graphs_pred_np,2) # plt.show() # test = utils_np.data_dicts_to_graphs_tuple(visualization_true_nodes_rollout_9_ge[example_number]) # first_six_graphs_np = utils_np.get_graph(test, slice(0, 5)) # plot_graphs_tuple_np(first_six_graphs_np,1) # first_six_graphs_np = utils_np.get_graph(test, slice(5, 10)) # plot_graphs_tuple_np(first_six_graphs_np,2) # plt.show() for example_number in range(0, 5): # first 10 examples test = utils_np.data_dicts_to_graphs_tuple( visualization_true_nodes_rollout_9_ge[example_number]) test2 = utils_np.data_dicts_to_graphs_tuple( prediction_over_time[example_number]) first_six_graphs_pred_np = utils_np.get_graph(test, slice(0, 10)) plot_graphs_tuple_np(first_six_graphs_pred_np, 1) first_six_graphs_pred_np = utils_np.get_graph(test2, slice(0, 10)) plot_graphs_tuple_np(first_six_graphs_pred_np, 2) plt.show() for example_number in range(40, 50): # first 10 examples test = utils_np.data_dicts_to_graphs_tuple( visualization_true_nodes_rollout_9_ge[example_number]) test2 = utils_np.data_dicts_to_graphs_tuple( prediction_over_time[example_number]) first_six_graphs_pred_np = utils_np.get_graph(test, slice(0, 10)) plot_graphs_tuple_np(first_six_graphs_pred_np, 1) first_six_graphs_pred_np = utils_np.get_graph(test2, slice(0, 10)) plot_graphs_tuple_np(first_six_graphs_pred_np, 2) plt.show()