def create_feed_dict(rand, batch_size, num_nodes_min_max, theta, input_ph,
target_ph):
"""Creates placeholders for the model training and evaluation.
Args:
rand: A random seed (np.RandomState instance).
batch_size: Total number of graphs per batch.
num_nodes_min_max: A 2-tuple with the [lower, upper) number of nodes per
graph. The number of nodes for a graph is uniformly sampled within this
range.
theta: A `float` threshold parameters for the geographic threshold graph's
threshold. Default= the number of nodes.
input_ph: The input graph's placeholders, as a graph namedtuple.
target_ph: The target graph's placeholders, as a graph namedtuple.
Returns:
feed_dict: The feed `dict` of input and target placeholders and data.
raw_graphs: The `dict` of raw networkx graphs.
"""
inputs, targets, raw_graphs = generate_networkx_graphs(
rand, batch_size, num_nodes_min_max, theta)
input_graphs = utils_np.networkxs_to_graphs_tuple(inputs)
target_graphs = utils_np.networkxs_to_graphs_tuple(targets)
feed_dict = {input_ph: input_graphs, target_ph: target_graphs}
return feed_dict, raw_graphs
def make_x_y(ll,yll,token,tokeny):#xll yll token=x_str2id tokeny=y_str2id
g = nx.Graph()
gy=nx.Graph()
n=len(ll)
#加节点 特征
for ni,char in enumerate(ll):
g.add_node(ni,features=[token[char]],char=char)
y=yll[ni]
if y=='unlabel':
y='PAD'
gy.add_node(ni,features=[tokeny[y]],char=y)
#加边
for ii in range(n)[:-1]:
g.add_edge(ii,ii+1,features=[0])
g.add_edge(ii+1, ii,features=[0])
#
gy.add_edge(ii,ii+1,features=[0])
gy.add_edge(ii+1, ii, features=[0])
# global
g.graph['features']=0
gy.graph['features']=0
### 增加 4 个triggerRoot node
g.add_node(n,features=[token['triggerRoot']],char='triggerRoot')
g.add_node(n+1,features=[token['triggerRoot']],char='triggerRoot')
g.add_node(n + 2, features=[token['triggerRoot']], char='triggerRoot')
g.add_node(n + 3, features=[token['triggerRoot']], char='triggerRoot')
#### ygraph 增加4个 pad
gy.add_node(n,features=[0],char='PAD')
gy.add_node(n+1,features=[0],char='PAD')
gy.add_node(n + 2, features=[0], char='PAD')
gy.add_node(n + 3, features=[0], char='PAD')
for nod in g.nodes(data=True):
nid,feadic=nod
if nid in [n,n+1,n+2,n+3]:continue # 是triggerRoot节点
###
if feadic['char'] not in event_mention_typ:continue # 额外的triggerRoot 只和event mention链接
g.add_edge(nid,n,features=[0])
g.add_edge(n, nid, features=[0])
#
g.add_edge(n+1, nid, features=[0])
g.add_edge(nid,n+1, features=[0])
g.add_edge(n + 2, nid, features=[0])
g.add_edge(nid, n + 2, features=[0])
g.add_edge(n + 3, nid, features=[0])
g.add_edge(nid, n + 3, features=[0])
#print ('')
gtx = utils_np.networkxs_to_graphs_tuple([g])
gty=utils_np.networkxs_to_graphs_tuple([gy])
#print ('')
return g,gy,gtx,gty
def test_nested_features(self):
graph_0 = utils_np.networkxs_to_graphs_tuple(
[_generate_graph(0, 3),
_generate_graph(1, 2)])
graph_1 = utils_np.networkxs_to_graphs_tuple([_generate_graph(2, 2)])
graph_2 = utils_np.networkxs_to_graphs_tuple([_generate_graph(3, 3)])
graphs_ = [
gr.map(tf.convert_to_tensor, graphs.ALL_FIELDS)
for gr in [graph_0, graph_1, graph_2]
]
def _create_nested_fields(graphs_tuple):
new_nodes = ({
"a": graphs_tuple.nodes,
"b": [graphs_tuple.nodes + 1, graphs_tuple.nodes + 2]
}, )
new_edges = [{
"c": graphs_tuple.edges + 5,
"d": (graphs_tuple.edges + 1, graphs_tuple.edges + 3),
}]
new_globals = []
return graphs_tuple.replace(nodes=new_nodes,
edges=new_edges,
globals=new_globals)
graphs_ = [_create_nested_fields(gr) for gr in graphs_]
concat_graph = utils_tf.concat(graphs_, axis=0)
actual_nodes = concat_graph.nodes
actual_edges = concat_graph.edges
actual_globals = concat_graph.globals
expected_nodes = tree.map_structure(lambda *x: tf.concat(x, axis=0),
*[gr.nodes for gr in graphs_])
expected_edges = tree.map_structure(lambda *x: tf.concat(x, axis=0),
*[gr.edges for gr in graphs_])
expected_globals = tree.map_structure(lambda *x: tf.concat(x, axis=0),
*[gr.globals for gr in graphs_])
tree.assert_same_structure(expected_nodes, actual_nodes)
tree.assert_same_structure(expected_edges, actual_edges)
tree.assert_same_structure(expected_globals, actual_globals)
tree.map_structure(self.assertAllEqual, expected_nodes, actual_nodes)
tree.map_structure(self.assertAllEqual, expected_edges, actual_edges)
tree.map_structure(self.assertAllEqual, expected_globals,
actual_globals)
# Borrowed from `test_concat_first_axis`:
self.assertAllEqual(np.array([3, 2, 2, 3]), concat_graph.n_node)
self.assertAllEqual(np.array([2, 1, 1, 2]), concat_graph.n_edge)
self.assertAllEqual(np.array([1, 2, 4, 6, 8, 9]), concat_graph.senders)
self.assertAllEqual(np.array([0, 0, 3, 5, 7, 7]),
concat_graph.receivers)
def test_graph_network(tf_session):
logdir = os.path.join(TEST_FOLDER, 'test_logdir/with_trace')
os.makedirs(logdir, exist_ok=True)
starcluster_graphs_nx, gaia_graphs_nx = make_example_graphs(
2,
num_stars=10,
sc_edge_size=2,
sc_node_size=12,
sc_global_size=5,
g_edge_size=3,
g_node_size=13,
g_global_size=6)
with tf_session.graph.as_default():
sc_pl = placeholders_from_networkxs(starcluster_graphs_nx,
force_dynamic_num_graphs=False)
g_pl = placeholders_from_networkxs(gaia_graphs_nx,
force_dynamic_num_graphs=False)
sc_graphtuple = networkxs_to_graphs_tuple(starcluster_graphs_nx)
g_graphtuple = networkxs_to_graphs_tuple(gaia_graphs_nx)
encoded_size = 7
t_network = StarClusterTNetwork(encoded_size,
sc_encoder_latent_size=16,
sc_encoder_num_layers=2,
sc_decoder_latent_size=16,
sc_decoder_num_layers=2,
g_encoder_latent_size=16,
g_encoder_num_layers=2)
t_network_output = t_network(g_pl,
sc_pl,
num_samples=2,
num_processing_steps=1)
summary = tf.summary.merge_all()
writer = tf.compat.v1.summary.FileWriter(logdir,
tf_session.graph,
session=tf_session)
tf_session.run(tf.global_variables_initializer())
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
t_network_output_res, summary_eval = tf_session.run(
[t_network_output, summary],
feed_dict={
sc_pl: sc_graphtuple,
g_pl: g_graphtuple
},
options=run_options,
run_metadata=run_metadata)
writer.add_run_metadata(run_metadata, 'step%d' % 0)
writer.add_summary(summary_eval, 0)
def create_feed_dict_yr(rand, batch_size, num_nodes_min_max, theta, input_ph,
target_ph):
"""Creates placeholders for the model training and evaluation.
Args:
rand: A random seed (np.RandomState instance).
batch_size: Total number of graphs per batch.
num_nodes_min_max: A 2-tuple with the [lower, upper) number of nodes per
graph. The number of nodes for a graph is uniformly sampled within this
range.
theta: A `float` threshold parameters for the geographic threshold graph's
threshold. Default= the number of nodes.
input_ph: The input graph's placeholders, as a graph namedtuple.
target_ph: The target graph's placeholders, as a graph namedtuple.
Returns:
feed_dict: The feed `dict` of input and target placeholders and data.
raw_graphs: The `dict` of raw networkx graphs.
"""
inputs, targets, raw_graphs = generate_networkx_graphs(
rand, batch_size, num_nodes_min_max, theta)
input_graphs = utils_np.networkxs_to_graphs_tuple(inputs)
target_graphs = utils_np.networkxs_to_graphs_tuple(targets)
# for k in all_fields:
# input_graphs.replace(k=tf.constant(input_graphs.k))
input_graphs = input_graphs.replace(edges=tf.constant(input_graphs.edges))
input_graphs = input_graphs.replace(
globals=tf.constant(input_graphs.globals))
input_graphs = input_graphs.replace(nodes=tf.constant(input_graphs.nodes))
input_graphs = input_graphs.replace(
n_edge=tf.constant(input_graphs.n_edge))
input_graphs = input_graphs.replace(
n_node=tf.constant(input_graphs.n_node))
input_graphs = input_graphs.replace(
receivers=tf.constant(input_graphs.receivers))
input_graphs = input_graphs.replace(
senders=tf.constant(input_graphs.senders))
target_graphs = target_graphs.replace(
edges=tf.constant(target_graphs.edges))
target_graphs = target_graphs.replace(
globals=tf.constant(target_graphs.globals))
target_graphs = target_graphs.replace(
nodes=tf.constant(target_graphs.nodes))
target_graphs = target_graphs.replace(
n_edge=tf.constant(target_graphs.n_edge))
target_graphs = target_graphs.replace(
n_node=tf.constant(target_graphs.n_node))
target_graphs = target_graphs.replace(
receivers=tf.constant(target_graphs.receivers))
target_graphs = target_graphs.replace(
senders=tf.constant(target_graphs.senders))
return [input_graphs, target_graphs]
def create_feed_dict(generator,
batch_size,
input_ph,
target_ph,
is_trained=True):
inputs, targets = generator(batch_size, is_trained)
input_graphs = utils_np.networkxs_to_graphs_tuple(inputs)
target_graphs = utils_np.networkxs_to_graphs_tuple(targets)
feed_dict = {input_ph: input_graphs, target_ph: target_graphs}
return feed_dict
def test_networkxs_to_graphs_tuple_raises_key_error(self):
"""If the "features" field is not present in the nodes or edges."""
graph_nx = _single_data_dict_to_networkx(self.graphs_dicts_in[-1])
first_node = list(graph_nx.nodes(data=True))[0]
del first_node[1]["features"]
with self.assertRaisesRegexp(
KeyError, "This could be due to the node having been silently added"):
utils_np.networkxs_to_graphs_tuple([graph_nx])
graph_nx = _single_data_dict_to_networkx(self.graphs_dicts_in[-1])
first_edge = list(graph_nx.edges(data=True))[0]
del first_edge[2]["features"]
with self.assertRaises(KeyError):
utils_np.networkxs_to_graphs_tuple([graph_nx])
def test_networkxs_to_graphs_tuple_raises_assertion_error(self):
"""Either all nodes (resp. edges) should have features, or none of them."""
graph_nx = _single_data_dict_to_networkx(self.graphs_dicts_in[-1])
first_node = list(graph_nx.nodes(data=True))[0]
first_node[1]["features"] = None
with self.assertRaisesRegexp(
ValueError, "Either all the nodes should have features"):
utils_np.networkxs_to_graphs_tuple([graph_nx])
graph_nx = _single_data_dict_to_networkx(self.graphs_dicts_in[-1])
first_edge = list(graph_nx.edges(data=True))[0]
first_edge[2]["features"] = None
with self.assertRaisesRegexp(
ValueError, "Either all the edges should have features"):
utils_np.networkxs_to_graphs_tuple([graph_nx])
def act(self,defState, defNode, eps):
"""
parse defstate to the input format for feed_dict
evaulate network to give action
"""
if (defState.nodes[defNode]["isDef"] != 1):
raise ValueError("def location doesn't match")
""" if episod is low or random value is les than epsilon
take random action
"""
if (eps < OBSERVEEPS) or (random.random() < EPSILON):
validActions = list(defState.out_edges([defNode]))
a = self.random.randint(0, len(validActions),size=1)[0]
return validActions[a]
gin = self._gtmp2intmp(defState)
test_values = self.sess.run({
"outputs":self.output_ops_tr
}, feed_dict={self.inputPh: utils_np.networkxs_to_graphs_tuple([gin])})
outg = utils_np.graphs_tuple_to_networkxs(test_values["outputs"][-1])[0]
outg = nx.DiGraph(outg)
self.outg = outg
validActions = list(defState.out_edges([defNode]))
qdict = dict()
for e in validActions:
qdict[e] = outg.get_edge_data(*e)["features"][0]
return max(qdict, key=qdict.get)
def test_feed_data(self):
networkx = [_generate_graph(batch_index) for batch_index in range(16)]
placeholders = utils_tf.placeholders_from_networkxs(
networkx, force_dynamic_num_graphs=True)
# Does not need to be the same size
networkxs = [_generate_graph(batch_index) for batch_index in range(2)]
with self.test_session() as sess:
output = sess.run(
placeholders,
utils_tf.get_feed_dict(placeholders,
utils_np.networkxs_to_graphs_tuple(networkxs)))
self.assertAllEqual(
np.array([[0, 0], [1, 0], [2, 0], [3, 0], [0, 1], [1, 1], [2, 1],
[3, 1]]), output.nodes)
self.assertEqual(np.float32, output.nodes.dtype)
self.assertAllEqual(np.array([[0], [1]]), output.globals)
self.assertEqual(np.float32, output.globals.dtype)
sorted_edges_content = sorted(
[(x, y, z)
for x, y, z in zip(output.receivers, output.senders, output.edges)])
self.assertAllEqual([0, 0, 1, 4, 4, 5],
[x[0] for x in sorted_edges_content])
self.assertAllEqual([1, 2, 3, 5, 6, 7],
[x[1] for x in sorted_edges_content])
self.assertEqual(np.float64, output.edges.dtype)
self.assertAllEqual(
np.array([[0, 1, 0], [1, 2, 0], [2, 3, 0], [0, 1, 1], [1, 2, 1],
[2, 3, 1]]), [x[2] for x in sorted_edges_content])
def create_feed_dict(input_ph, target_ph, inputs, targets):
"""Creates the feed dict for the placeholders for the model training and evaluation.
Args:
input_ph: The input graph's placeholders, as a graph namedtuple.
target_ph: The target graph's placeholders, as a graph namedtuple.
inputs: The input graphs
targets: The target graphs
Returns:
feed_dict: The feed `dict` of input and target placeholders and data.
"""
input_graphs = utils_np.networkxs_to_graphs_tuple(inputs)
target_graphs = utils_np.networkxs_to_graphs_tuple(targets)
feed_dict = {input_ph: input_graphs, target_ph: target_graphs}
return feed_dict
def test_raise_all_or_no_nones(self, none_field):
graph_0 = utils_np.networkxs_to_graphs_tuple(
[_generate_graph(0, 3),
_generate_graph(1, 2)])
graph_1 = utils_np.networkxs_to_graphs_tuple([_generate_graph(2, 2)])
graph_2 = utils_np.networkxs_to_graphs_tuple([_generate_graph(3, 3)])
graphs_ = [
gr.map(tf.convert_to_tensor, graphs.ALL_FIELDS)
for gr in [graph_0, graph_1, graph_2]
]
graphs_[1] = graphs_[1].replace(**{none_field: None})
with self.assertRaisesRegex(
ValueError,
"Different set of keys found when iterating over data dictionaries."
):
utils_tf.concat(graphs_, axis=0)
def generate_example_random_choice(positions, properties, k=26, plot=False):
print('choice nn')
idx_list = np.arange(len(positions))
virtual_node_positions = positions[np.random.choice(idx_list, 1000, replace=False)]
kdtree = cKDTree(virtual_node_positions)
dist, indices = kdtree.query(positions)
virtual_properties = np.zeros((len(np.bincount(indices)), len(properties[0])))
mean_sum = [lambda x: np.bincount(indices, weights=x) / np.maximum(1., np.bincount(indices)), # mean
lambda x: np.bincount(indices, weights=x)] # sum
mean_sum_enc = [0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1]
for p, enc in zip(np.arange(len(properties[0])), mean_sum_enc):
virtual_properties[:, p] = mean_sum[enc](properties[:, p])
virtual_positions = virtual_properties[:, :3]
graph = nx.DiGraph()
kdtree = cKDTree(virtual_positions)
dist, idx = kdtree.query(virtual_positions, k=k + 1)
receivers = idx[:, 1:] # N,k
senders = np.arange(virtual_positions.shape[0]) # N
senders = np.tile(senders[:, None], [1, k]) # N,k
receivers = receivers.flatten()
senders = senders.flatten()
n_nodes = virtual_positions.shape[0]
pos = dict() # for plotting node positions.
edgelist = []
for node, feature, position in zip(np.arange(n_nodes), virtual_properties, virtual_positions):
graph.add_node(node, features=feature)
pos[node] = position[:2]
# edges = np.stack([senders, receivers], axis=-1) + sibling_node_offset
for u, v in zip(senders, receivers):
graph.add_edge(u, v, features=np.array([1., 0.]))
graph.add_edge(v, u, features=np.array([1., 0.]))
edgelist.append((u, v))
edgelist.append((v, u))
graph.graph["features"] = np.array([0.])
# plotting
print('len(pos) = {}\nlen(edgelist) = {}'.format(len(pos), len(edgelist)))
if plot:
fig, ax = plt.subplots(1, 1, figsize=(20, 20))
draw(graph, ax=ax, pos=pos, node_color='blue', edge_color='red', node_size=10, width=0.1)
image_dir = '/data2/hendrix/images/'
graph_image_idx = len(glob.glob(os.path.join(image_dir, 'graph_image_*')))
plt.savefig(os.path.join(image_dir, 'graph_image_{}'.format(graph_image_idx)))
return networkxs_to_graphs_tuple([graph],
node_shape_hint=[virtual_positions.shape[1] + virtual_properties.shape[1]],
edge_shape_hint=[2])
def test_concat_last_axis(self):
graph0 = utils_np.networkxs_to_graphs_tuple(
[_generate_graph(0, 3), _generate_graph(1, 2)])
graph1 = utils_np.networkxs_to_graphs_tuple(
[_generate_graph(2, 3), _generate_graph(3, 2)])
graph0 = graph0.map(tf.convert_to_tensor, graphs.ALL_FIELDS)
graph1 = graph1.map(tf.convert_to_tensor, graphs.ALL_FIELDS)
concat_graph = utils_tf.concat([graph0, graph1], axis=-1)
self.assertAllEqual(
np.array([[0, 0, 0, 2], [1, 0, 1, 2], [2, 0, 2, 2], [0, 1, 0, 3],
[1, 1, 1, 3]]), concat_graph.nodes)
self.assertAllEqual(
np.array([[0, 1, 0, 0, 1, 2], [1, 2, 0, 1, 2, 2], [0, 1, 1, 0, 1, 3]]),
concat_graph.edges)
self.assertAllEqual(np.array([3, 2]), concat_graph.n_node)
self.assertAllEqual(np.array([2, 1]), concat_graph.n_edge)
self.assertAllEqual(np.array([1, 2, 4]), concat_graph.senders)
self.assertAllEqual(np.array([0, 0, 3]), concat_graph.receivers)
self.assertAllEqual(np.array([[0, 2], [1, 3]]), concat_graph.globals)
def create_feed_dict(all_db, rand, batch_size, input_ph, target_ph, dataset, graphcache):
"""Creates placeholders for the model training and evaluation.
Args:
rand: A random seed (np.RandomState instance).
batch_size: Total number of graphs per batch.
input_ph: The input graph's placeholders, as a graph namedtuple.
target_ph: The target graph's placeholders, as a graph namedtuple.
dataset: 'train', 'val', 'test'
Returns:
feed_dict: The feed `dict` of input and target placeholders and data.
raw_graphs: The `dict` of raw networkx graphs.
"""
inputs, targets, raw_graphs, selids = generate_networkx_graphs(graphcache, all_db, rand, batch_size, dataset)
input_graphs = utils_np.networkxs_to_graphs_tuple(inputs)
target_graphs = utils_np.networkxs_to_graphs_tuple(targets)
feed_dict = {input_ph: input_graphs, target_ph: target_graphs}
return feed_dict, raw_graphs, selids
def test_get_feed_dict_raises(self, none_fields):
networkxs = [_generate_graph(batch_index) for batch_index in range(16)]
placeholders = utils_tf.placeholders_from_networkxs(networkxs)
feed_values = utils_np.networkxs_to_graphs_tuple(networkxs)
with self.assertRaisesRegexp(ValueError, ""):
utils_tf.get_feed_dict(
placeholders.map(lambda _: None, none_fields), feed_values)
with self.assertRaisesRegexp(ValueError, ""):
utils_tf.get_feed_dict(placeholders,
feed_values.map(lambda _: None, none_fields))
def test_concat_first_axis(self, none_fields):
graph_0 = utils_np.networkxs_to_graphs_tuple(
[_generate_graph(0, 3),
_generate_graph(1, 2)])
graph_1 = utils_np.networkxs_to_graphs_tuple([_generate_graph(2, 2)])
graph_2 = utils_np.networkxs_to_graphs_tuple([_generate_graph(3, 3)])
graphs_ = [
gr.map(tf.convert_to_tensor, graphs.ALL_FIELDS)
for gr in [graph_0, graph_1, graph_2]
]
graphs_ = [gr.map(lambda _: None, none_fields) for gr in graphs_]
concat_graph = utils_tf.concat(graphs_, axis=0)
for none_field in none_fields:
self.assertEqual(None, getattr(concat_graph, none_field))
concat_graph = concat_graph.map(tf.no_op, none_fields)
with self.test_session() as sess:
concat_graph = sess.run(concat_graph)
if "nodes" not in none_fields:
self.assertAllEqual(np.array([0, 1, 2, 0, 1, 0, 1, 0, 1, 2]),
[x[0] for x in concat_graph.nodes])
self.assertAllEqual(np.array([0, 0, 0, 1, 1, 2, 2, 3, 3, 3]),
[x[1] for x in concat_graph.nodes])
if "edges" not in none_fields:
self.assertAllEqual(np.array([0, 1, 0, 0, 0, 1]),
[x[0] for x in concat_graph.edges])
self.assertAllEqual(np.array([0, 0, 1, 2, 3, 3]),
[x[2] for x in concat_graph.edges])
self.assertAllEqual(np.array([3, 2, 2, 3]), concat_graph.n_node)
self.assertAllEqual(np.array([2, 1, 1, 2]), concat_graph.n_edge)
if "senders" not in none_fields:
# [1, 2], [1], [1], [1, 2] and 3, 2, 2, 3 nodes
# So we are summing [1, 2, 1, 1, 2] with [0, 0, 3, 5, 7, 7]
self.assertAllEqual(np.array([1, 2, 4, 6, 8, 9]),
concat_graph.senders)
if "receivers" not in none_fields:
# [0, 0], [0], [0], [0, 0] and 3, 2, 2, 3 nodes
# So we are summing [0, 0, 0, 0, 0, 0] with [0, 0, 3, 5, 7, 7]
self.assertAllEqual(np.array([0, 0, 3, 5, 7, 7]),
concat_graph.receivers)
if "globals" not in none_fields:
self.assertAllEqual(np.array([[0], [1], [2], [3]]),
concat_graph.globals)
def next(self, raw_graphs=False):
if self.indx + self.batch_size > self.len:
self.indx = 0
if self.shuffle:
np.random.shuffle(self.indices)
indx_range = self.indices[range(self.indx, self.indx + self.batch_size)]
graphs = [self.graphs[i] for i in indx_range]
ys = [self.y_matrices[i] for i in indx_range]
targets = copy.deepcopy(graphs)
for i in range(len(ys)):
graphs[i].graph['features'] = np.array([0, 0]).astype(np.float)
targets[i].graph['features'] = np.array(ys[i]).astype(np.float)
self.indx += self.batch_size
if raw_graphs == True:
return graphs, targets
else:
return utils_np.networkxs_to_graphs_tuple(graphs), utils_np.networkxs_to_graphs_tuple(targets)
def infer(self, input_graphs, target_graphs):
reload_file = Path(self._reload_fle)
input_ph, target_ph = create_placeholders(input_graphs, target_graphs)
input_ph, target_ph = make_all_runnable_in_session(input_ph, target_ph)
output_ops_ge = self._model(input_ph, self._num_processing_steps_ge)
saver = tf.train.import_meta_graph(reload_file.as_posix() + '.meta')
sess = tf.Session()
sess.run(tf.global_variables_initializer())
tf.reset_default_graph()
with sess.as_default():
if not reload_file.is_dir():
saver.restore(sess, reload_file.as_posix())
else:
print("no file found, restoring failed")
input_graphs_tuple = utils_np.networkxs_to_graphs_tuple(
input_graphs)
target_graphs_tuple = utils_np.networkxs_to_graphs_tuple(
target_graphs)
feed_dict = {
input_ph: input_graphs_tuple,
target_ph: target_graphs_tuple,
}
test_values = sess.run(
{
"target": target_ph,
"outputs": output_ops_ge,
},
feed_dict=feed_dict)
correct_ge, solved_ge = existence_accuracy(
test_values["target"],
test_values["outputs"][-1],
use_edges=False)
testing_info = 0, 0, 0, 0, [correct_ge], 0, [solved_ge]
return test_values, testing_info
def test_networkxs_to_graphs_tuple(self):
graph0 = utils_np.data_dicts_to_graphs_tuple(self.graphs_dicts_in)
graph_nxs = []
for data_dict in self.graphs_dicts_in:
graph_nxs.append(_single_data_dict_to_networkx(data_dict))
hints = {
"edge_shape_hint": data_dict["edges"].shape[1:],
"node_shape_hint": data_dict["nodes"].shape[1:],
"data_type_hint": data_dict["nodes"].dtype,
}
graph = utils_np.networkxs_to_graphs_tuple(graph_nxs, **hints)
self._assert_graph_equals_np(graph0, graph, force_edges_ordering=True)
def create_feed_dict(sources, targets, source_ph, target_ph):
"""Creates placeholders for the model training and evaluation.
Args:
rand: A random seed (np.RandomState instance).
batch_size: Total number of graphs per batch.
min_max_nodes: A 2-tuple with the [lower, upper) number of nodes per
graph. The number of nodes for a graph is uniformly sampled within this
range.
geo_density: A `float` threshold parameters for the geographic threshold graph's
threshold. Default= the number of nodes.
source_ph: The source graph's placeholders, as a graph namedtuple.
target_ph: The target graph's placeholders, as a graph namedtuple.
Returns:
feed_dict: The feed `dict` of source and target placeholders and data.
"""
source_graphs = utils_np.networkxs_to_graphs_tuple(sources)
target_graphs = utils_np.networkxs_to_graphs_tuple(targets)
feed_dict = {source_ph: source_graphs, target_ph: target_graphs}
return feed_dict
def create_feed_dict(input_ph,
target_ph,
input_graphs,
target_graphs,
batch_processing=True):
if batch_processing:
input_graphs = input_graphs
target_graphs = target_graphs
else:
input_graphs = [input_graphs]
target_graphs = [target_graphs]
input_tuple = utils_np.networkxs_to_graphs_tuple(input_graphs)
target_tuple = utils_np.networkxs_to_graphs_tuple(target_graphs)
input_dct = utils_tf.get_feed_dict(input_ph, input_tuple)
target_dct = utils_tf.get_feed_dict(target_ph, target_tuple)
input_ph_runnable, target_ph_runnable = make_all_runnable_in_session(
input_ph, target_ph)
return input_ph_runnable, target_ph_runnable, {**input_dct, **target_dct}
def create_feed_dict(rand, batch_size, raw_input_graphs, raw_target_graphs,
edge_permutations, input_ph, target_ph, weight_ph,
edge_rel_weights, sample_ids, replace):
"""Creates placeholders for the model training and evaluation."""
# Create some example data for inspecting the vector sizes.
inputs, targets = generate_networkx_graphs(rand, batch_size,
raw_input_graphs,
raw_target_graphs,
edge_permutations, sample_ids,
replace)
input_graphs = utils_np.networkxs_to_graphs_tuple(inputs)
target_graphs = utils_np.networkxs_to_graphs_tuple(targets)
graph_edge_weights = edge_rel_weights[target_graphs[1][:,
-1].astype(np.int)]
norm_graph_edge_weights = graph_edge_weights / (
graph_edge_weights[graph_edge_weights > 0].mean())
feed_dict = {
input_ph: input_graphs,
target_ph: target_graphs,
weight_ph: norm_graph_edge_weights
}
return feed_dict
def generate_example_nn(positions, properties, k=1, plot=False):
"""
Generate a k-nn graph from positions.
Args:
positions: [num_points, 3] positions used for graph constrution.
properties: [num_points, F0,...,Fd] each node will have these properties of shape [F0,...,Fd]
k: int, k nearest neighbours are connected.
plot: whether to plot graph.
Returns: GraphTuple
"""
graph = nx.OrderedMultiDiGraph()
kdtree = cKDTree(positions)
dist, idx = kdtree.query(positions, k=k + 1)
receivers = idx[:, 1:] #N,k
senders = np.arange(positions.shape[0]) #N
senders = np.tile(senders[:, None], [1, k]) #N,k
receivers = receivers.flatten()
senders = senders.flatten()
n_nodes = positions.shape[0]
pos = dict() # for plotting node positions.
edgelist = []
for node, feature, position in zip(np.arange(n_nodes), properties,
positions):
graph.add_node(node, features=feature)
pos[node] = position[:2]
# edges = np.stack([senders, receivers], axis=-1) + sibling_node_offset
for u, v in zip(senders, receivers):
graph.add_edge(u, v, features=None)
graph.add_edge(v, u, features=None)
edgelist.append((u, v))
edgelist.append((v, u))
graph.graph['features'] = None
# plotting
if plot:
fig, ax = plt.subplots(1, 1, figsize=(12, 12))
draw(graph, ax=ax, pos=pos, node_color='green', edge_color='red')
plt.show()
return networkxs_to_graphs_tuple(
[graph], node_shape_hint=[positions.shape[1] + properties.shape[1]])
def placeholders_from_networkxs(graph_nxs,
node_shape_hint=None,
edge_shape_hint=None,
data_type_hint=tf.float32,
force_dynamic_num_graphs=True,
name="placeholders_from_networkxs"):
"""Constructs placeholders compatible with a list of networkx instances.
Given a list of networkxs instances, constructs placeholders compatible with
the shape of those graphs.
The networkx graph should be set up such that, for fixed shapes `node_shape`,
`edge_shape` and `global_shape`:
- `graph_nx.nodes(data=True)[i][-1]["features"]` is, for any node index i, a
tensor of shape `node_shape`, or `None`;
- `graph_nx.edges(data=True)[i][-1]["features"]` is, for any edge index i, a
tensor of shape `edge_shape`, or `None`;
- `graph_nx.edges(data=True)[i][-1]["index"]`, if present, defines the order
in which the edges will be sorted in the resulting `data_dict`;
- `graph_nx.graph["features"] is a tensor of shape `global_shape` or `None`.
Args:
graph_nxs: A container of `networkx.MultiDiGraph`s.
node_shape_hint: (iterable of `int` or `None`, default=`None`) If the graph
does not contain nodes, the trailing shape for the created `NODES` field.
If `None` (the default), this field is left `None`. This is not used if
`graph_nx` contains at least one node.
edge_shape_hint: (iterable of `int` or `None`, default=`None`) If the graph
does not contain edges, the trailing shape for the created `EDGES` field.
If `None` (the default), this field is left `None`. This is not used if
`graph_nx` contains at least one edge.
data_type_hint: (numpy dtype, default=`np.float32`) If the `NODES` or
`EDGES` fields are autocompleted, their type.
force_dynamic_num_graphs: A `bool` that forces the batch dimension to be
dynamic. Defaults to `True`.
name: (string, optional) A name for the operation.
Returns:
An instance of `graphs.GraphTuple` placeholders compatible with the
dimensions of the graph_nxs.
"""
with tf.name_scope(name):
graph = utils_np.networkxs_to_graphs_tuple(
graph_nxs, node_shape_hint, edge_shape_hint,
data_type_hint.as_numpy_dtype())
return _placeholders_from_graphs_tuple(
graph, force_dynamic_num_graphs=force_dynamic_num_graphs)
def test_feed_data_no_nodes(self):
networkx = [
_generate_graph(batch_index, n_nodes=0, add_edges=False)
for batch_index in range(16)
]
placeholders = utils_tf.placeholders_from_networkxs(
networkx, force_dynamic_num_graphs=True)
# Does not need to be the same size
networkxs = [
_generate_graph(batch_index, n_nodes=0, add_edges=False)
for batch_index in range(2)
]
self.assertEqual(None, placeholders.nodes)
self.assertEqual(None, placeholders.edges)
with self.test_session() as sess:
output = sess.run(
placeholders.replace(nodes=tf.no_op(), edges=tf.no_op()),
utils_tf.get_feed_dict(placeholders,
utils_np.networkxs_to_graphs_tuple(networkxs)))
self.assertAllEqual(np.array([[0], [1]]), output.globals)
self.assertEqual(np.float32, output.globals.dtype)
def _get_observation(self):
graph = ObservationBuilder(features=self._features).get_observation(
self.env)
gt = utils_np.networkxs_to_graphs_tuple([graph])
# build action indices here to make sure the indices matches the
# one the network is seeing
self.actions = []
edges = gt.edges if gt.edges is not None else [] # may be None
for (u, v, d) in zip(gt.senders, gt.receivers, edges):
possible = d[POSSIBLE_IDX] == 1
if not possible:
continue
else:
source = graph.node[u]["represents"]
target = graph.node[v]["represents"]
timeslot = int(d[TIMESLOT_IDX])
self.actions.append((source, target, timeslot))
return gt
def next_batch(self, index):
graph_dicts = []
labels = []
for k, path in enumerate(
self.second_image_paths[self.batch_size *
index:self.batch_size * (index + 1)]):
temp_str = path.split('\\')[-1]
begin = temp_str.find('_')
end = temp_str.find('.')
label = self.alpha.index(temp_str[begin + 2:end])
img = cv2.imread(path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = img / 255.0
graph_dicts.append(self.deal_image(img))
labels.append(label)
return utils_np.networkxs_to_graphs_tuple(graph_dicts), np.eye(
len(self.alpha))[labels]
def test_batch_gen():
#['caveman_2', 'caveman_4']
gen_graph = GenerateDataGraph(type_dataset='caveman_4', num_perm=10)
epochs = 1
batch_size = 40
for epoch in range(epochs):
print("\n########## epoch " + str(epoch + 1) + " ##########")
gen_trainig = gen_graph.train_generator(batch_size=batch_size)
counter = 0
for gt_graph, set_feature, in_graph in gen_trainig:
print("---- batch ----")
#print("gt_graph.shape: ", gt_graph.shape)
print("set_feature.shape: ", set_feature.shape)
print("in_graph.shape: ", in_graph.shape)
#draw_graph(G_arr=gt_graph, row=2, col=2, pos=set_feature, fname='comm/comm_'+str(counter))
for g in gt_graph:
nx.set_node_attributes(G=g, name="features", values=0)
nx.set_edge_attributes(G=g, name="features", values=0)
graph_tuple = utils_np.networkxs_to_graphs_tuple(gt_graph)
print(graph_tuple)
#for k in range(len(gt_graph)):
# counter += 1
# save_graph(name='comm/comm_'+str(counter), points_coord=set_feature[k], adj=gt_graph[k], dim=2)
break
def generate_example(positions, properties, k_mean=26, plot=False):
"""
Generate a geometric graph from positions.
Args:
positions: [num_points, 3] positions used for graph constrution.
properties: [num_points, F0,...,Fd] each node will have these properties of shape [F0,...,Fd]
k_mean: float
plot: whether to plot graph.
Returns: GraphTuple
"""
graph = nx.DiGraph()
sibling_edgelist = []
parent_edgelist = []
pos = dict() # for plotting node positions.
real_nodes = list(np.arange(positions.shape[0]))
while positions.shape[0] > 1:
# n_nodes, n_nodes
dist = np.linalg.norm(positions[:, None, :] - positions[None, :, :], axis=-1)
opt_screen_length = find_screen_length(dist, k_mean)
print("Found optimal screening length {}".format(opt_screen_length))
distance_matrix_no_loops = np.where(dist == 0., np.inf, dist)
A = distance_matrix_no_loops < opt_screen_length
senders, receivers = np.where(A)
n_edge = senders.size
# [1,0] for siblings, [0,1] for parent-child
sibling_edges = np.tile([[1., 0.]], [n_edge, 1])
# num_points, F0,...Fd
# if positions is to be part of features then this should already be set in properties.
# We don't concatentate here. Mainly because properties could be an image, etc.
sibling_nodes = properties
n_nodes = sibling_nodes.shape[0]
sibling_node_offset = len(graph.nodes)
for node, feature, position in zip(np.arange(sibling_node_offset, sibling_node_offset + n_nodes), sibling_nodes,
positions):
graph.add_node(node, features=feature)
pos[node] = position[:2]
# edges = np.stack([senders, receivers], axis=-1) + sibling_node_offset
for u, v in zip(senders + sibling_node_offset, receivers + sibling_node_offset):
graph.add_edge(u, v, features=np.array([1., 0.]))
graph.add_edge(v, u, features=np.array([1., 0.]))
sibling_edgelist.append((u, v))
sibling_edgelist.append((v, u))
# for virtual nodes
sibling_graph = GraphsTuple(nodes=None, # sibling_nodes,
edges=None,
senders=senders,
receivers=receivers,
globals=None,
n_node=np.array([n_nodes]),
n_edge=np.array([n_edge]))
sibling_graph = graphs_tuple_to_networkxs(sibling_graph)[0]
# completely connect
connected_components = sorted(nx.connected_components(nx.Graph(sibling_graph)), key=len)
_positions = []
_properties = []
for connected_component in connected_components:
print("Found connected component {}".format(connected_component))
indices = list(sorted(list(connected_component)))
virtual_position, virtual_property = make_virtual_node(positions[indices, :], properties[indices, ...])
_positions.append(virtual_position)
_properties.append(virtual_property)
virtual_positions = np.stack(_positions, axis=0)
virtual_properties = np.stack(_properties, axis=0)
###
# add virutal nodes
# num_parents, 3+F
parent_nodes = virtual_properties
n_nodes = parent_nodes.shape[0]
parent_node_offset = len(graph.nodes)
parent_indices = np.arange(parent_node_offset, parent_node_offset + n_nodes)
# adding the nodes to global graph
for node, feature, virtual_position in zip(parent_indices, parent_nodes, virtual_positions):
graph.add_node(node, features=feature)
print("new virtual {}".format(node))
pos[node] = virtual_position[:2]
for parent_idx, connected_component in zip(parent_indices, connected_components):
child_node_indices = [idx + sibling_node_offset for idx in list(sorted(list(connected_component)))]
for child_node_idx in child_node_indices:
graph.add_edge(parent_idx, child_node_idx, features=np.array([0., 1.]))
graph.add_edge(child_node_idx, parent_idx, features=np.array([0., 1.]))
parent_edgelist.append((parent_idx, child_node_idx))
parent_edgelist.append((child_node_idx, parent_idx))
print("connecting {}<->{}".format(parent_idx, child_node_idx))
positions = virtual_positions
properties = virtual_properties
# plotting
virutal_nodes = list(set(graph.nodes) - set(real_nodes))
if plot:
fig, ax = plt.subplots(1, 1, figsize=(12, 12))
draw(graph, ax=ax, pos=pos, node_color='green', edgelist=[], nodelist=real_nodes)
draw(graph, ax=ax, pos=pos, node_color='purple', edgelist=[], nodelist=virutal_nodes)
draw(graph, ax=ax, pos=pos, edge_color='blue', edgelist=sibling_edgelist, nodelist=[])
draw(graph, ax=ax, pos=pos, edge_color='red', edgelist=parent_edgelist, nodelist=[])
plt.show()
return networkxs_to_graphs_tuple([graph],
node_shape_hint=[positions.shape[1] + properties.shape[1]],
edge_shape_hint=[2])
