def test_merge_nearby_nodes():
"""
--2 2
/ /
-- /
/ /
1 /
\ /
0-----------5 => 0------------5
/
3
|
4
"""
coords = [[ 0, 0],
[-1, 1],
[ 2, 6],
[-1, -1],
[-1, -2],
[ 5, 0]]
edges = [(0, 1), (1, 2), (0, 3), (3, 4), (0, 5)]
geo = GeoGraph(coords=dict(enumerate(coords)), data=edges)
geo.merge_nearby_nodes(radius=2.0)
assert geo.edges() == [(0, 2), (0, 5)],\
"nodes were not merged correctly"
def simple_nodes_disjoint_grid():
"""
return disjoint net plus nodes with fakes
fakes are associated with disjoint subnets
nodes by id (budget in parens)
(5) 0-------1 (5)
| |
+-+-+ +-+-+ <-- disjoint existing grid
Useful for testing treating existing grid as single grid
vs disjoint
"""
# setup grid
grid_coords = np.array([[-1.0, 0.0], [1.0, 0.0], [3.0, 0.0], [5.0, 0.0]])
grid = GeoGraph(gm.PROJ4_FLAT_EARTH, {'grid-' + str(n): c for n, c in
enumerate(grid_coords)})
nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()})
grid.add_edges_from([('grid-0', 'grid-1'), ('grid-2', 'grid-3')])
# setup input nodes
node_coords = np.array([[0.0, 1.0], [4.0, 1.0]])
nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords)))
budget_values = [5, 5]
nx.set_node_attributes(nodes, 'budget', dict(enumerate(budget_values)))
fakes = [2, 3]
return grid, nodes, fakes
def grid_and_non_grid():
"""
return networkplan GeoGraph with grid and non-grid components
0 2
| |
+-6-+ 1 3-4-5
(inf)
where node 3 is a fake node connecting node 0 to the grid
"""
node_coords = np.array([[0.0, 1.0],
[4.0, 0.0],
[4.0, 1.0],
[5.0, 0.0],
[6.0, 0.0],
[7.0, 0.0],
[0.0, 0.0]])
grid = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords)))
budget_values = [5, 5, 5, 5, 5, 5, np.inf]
nx.set_node_attributes(grid, 'budget', dict(enumerate(budget_values)))
grid.add_edges_from([(0, 6), (1, 2), (3, 4), (4, 5)])
return grid
def dataset_store_to_geograph(dataset_store):
"""
convenience function for converting a network stored in a dataset_store
into a GeoGraph
Args:
dataset_store containing a network
Returns:
GeoGraph representation of dataset_store network
TODO: determine projection from dataset_store?
"""
all_nodes = list(dataset_store.cycleNodes()) + \
list(dataset_store.cycleNodes(isFake=True))
# nodes in output GeoGraph are id'd from 0 to n (via enumerate)
np_to_nx_id = {node.id: i for i, node in enumerate(all_nodes)}
coords = [node.getCommonCoordinates() for node in all_nodes]
coords_dict = dict(enumerate(coords))
G = GeoGraph(coords=coords_dict)
# only set population, system and budget for now
# TODO: Do we need all from the output?
for i, node in enumerate(all_nodes):
if not node.is_fake:
properties = {
'budget': node.metric,
'population': node.output['demographics']['population count'],
'system': node.output['metric']['system']
}
G.node[i] = properties
def seg_to_nx_ids(seg):
"""
Return the networkx segment ids
"""
return (np_to_nx_id[seg.node1_id],
np_to_nx_id[seg.node2_id])
edges = [seg_to_nx_ids(s) for s in
dataset_store.cycleSegments(is_existing=False)]
edge_weights = {seg_to_nx_ids(s): s.weight for s in
dataset_store.cycleSegments(is_existing=False)}
edge_is_existing = {seg_to_nx_ids(s): s.is_existing for s in
dataset_store.cycleSegments(is_existing=False)}
edge_subnet_id = {seg_to_nx_ids(s): s.subnet_id for s in
dataset_store.cycleSegments(is_existing=False)}
G.add_edges_from(edges)
nx.set_edge_attributes(G, 'weight', edge_weights)
nx.set_edge_attributes(G, 'is_existing', edge_is_existing)
nx.set_edge_attributes(G, 'subnet_id', edge_subnet_id)
return G
def read_csv_geograph(csv_file, x_column="X", y_column="Y"):
"""
load nodes csv into GeoGraph (nodes only for now)
Args:
csv_file: nodal metrics csv file as string or file
x_column, y_column: col names to take x, y from
Returns:
GeoGraph of nodes including all attributes from input csv
"""
input_proj = read_csv_projection(csv_file)
header_row = 1 if input_proj else 0
# read in the csv
# NOTE: Convert x,y via float cast to preserve precision of input
# go back to beginning 1st
csv_file.seek(0)
metrics = pd.read_csv(csv_file,
header=header_row,
converters={
x_column: float,
y_column: float
})
coord_cols = [x_column, y_column]
assert all([hasattr(metrics, col) for col in coord_cols]), \
"metrics file does not contain coordinate columns {}, {}".\
format(x_column, y_column)
# Stack the coords
coords = np.column_stack(map(metrics.get, coord_cols))
# set default projection
if not input_proj:
if gm.is_in_lon_lat(coords):
input_proj = gm.PROJ4_LATLONG
else:
input_proj = gm.PROJ4_FLAT_EARTH
coords_dict = dict(enumerate(coords))
geo_nodes = GeoGraph(input_proj, coords_dict)
# populate the rest of the attributes
metrics_no_coords = metrics[metrics.columns.difference(coord_cols)]
for row in metrics_no_coords.iterrows():
index = row[0]
attrs = row[1].to_dict()
geo_nodes.node[index] = attrs
return geo_nodes
def _get_demand_nodes(self, input_proj=None):
"""
Converts the dataset_store metrics records to a GeoGraph of nodes
(prereq: _run_metric_model to populate store)
Args:
input_proj: projection of demand node coordinates
Returns:
GeoGraph: demand nodes as GeoGraph
"""
coords = [
node.getCommonCoordinates() for node in self.store.cycleNodes()
]
# set default projection
if not input_proj:
input_proj = self._get_default_proj4(coords)
# NOTE: Although dataset_store nodes id sequence starts at 1
# leave the GeoGraph ids 0 based because there are places in the
# network algorithm that assume 0 based coords
# This will be realigned later
coords_dict = {i: coord for i, coord in enumerate(coords)}
budget_dict = {
i: node.metric
for i, node in enumerate(self.store.cycleNodes())
}
geo_nodes = GeoGraph(input_proj, coords_dict)
nx.set_node_attributes(geo_nodes, 'budget', budget_dict)
return geo_nodes
def random_settlements(n):
coords = np.random.uniform(size=(n, 2))
# get all perm's of points (repetitions are ok here)
points_left = np.tile(coords, (len(coords), 1))
points_right = np.repeat(coords, len(coords), axis=0)
point_pairs = np.concatenate(
(points_left[:, np.newaxis], points_right[:, np.newaxis]), axis=1)
all_dists = gm.spherical_distance_haversine(point_pairs)
full_dist_matrix = all_dists.reshape(len(coords), len(coords))
zero_indices = (np.array(range(len(coords))) * (len(coords) + 1))
non_zero_dists = np.delete(all_dists, zero_indices).\
reshape((len(coords), len(coords) - 1))
# find all minimum distances
# apply min over ranges of the dist array
min_dists = np.min(non_zero_dists, axis=1)
# assign same median budget to all nodes
# outside a really degenerate case (all edges in line in shortest
# distance order...)
# this should ensure some "dead" nodes
budget_vals = np.repeat(np.median(min_dists), len(coords))
# build graph
graph = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(coords)))
nx.set_node_attributes(graph, 'budget', dict(enumerate(budget_vals)))
return graph, full_dist_matrix
def read_json_geograph(json_file):
"""
Args:
json_file: path to json file as string or file
Assumes the json is in networkx link-node format
"""
js = json.load(json_file)
g = json_graph.node_link_graph(js)
assert all([nd.has_key('coords') for nd in g.node.values()]),\
"json node-link graph must have nodes with coords for GeoGraph"
# get coords
coords = [v['coords'] for v in g.node.values()]
# set default projection
input_proj = ""
if gm.is_in_lon_lat(coords):
input_proj = gm.PROJ4_LATLONG
else:
input_proj = gm.PROJ4_FLAT_EARTH
coords_dict = {k: v['coords'] for k, v in g.node.items()}
# now get rid of 'coords' key,val for each node
for node in g.node.values():
node.pop('coords', None)
geo_nodes = GeoGraph(srs=input_proj, coords=coords_dict, data=g)
return geo_nodes
def load_nodes(filename="metrics.csv", x_column="X", y_column="Y"):
"""
load nodes csv into GeoGraph (nodes only)
Args:
filename: nodal metrics csv file
x_column, y_column: col names to take x, y from
Returns:
GeoGraph of nodes including all attributes from input csv
"""
input_proj = csv_projection(filename)
header_row = 1 if input_proj else 0
# read in the csv
# NOTE: Convert x,y via float cast to preserve precision of input
metrics = pd.read_csv(filename, header=header_row,
converters={x_column: float, y_column: float})
coord_cols = [x_column, y_column]
assert all([hasattr(metrics, col) for col in coord_cols]), \
"metrics file does not contain coordinate columns {}, {}".\
format(x_column, y_column)
# Stack the coords
coords = np.column_stack(map(metrics.get, coord_cols))
# set default projection
if not input_proj:
if gm.is_in_lon_lat(coords):
input_proj = gm.PROJ4_LATLONG
else:
input_proj = gm.PROJ4_FLAT_EARTH
coords_dict = dict(enumerate(coords))
geo_nodes = GeoGraph(input_proj, coords_dict)
# populate the rest of the attributes
metrics_no_coords = metrics[metrics.columns.difference(coord_cols)]
for row in metrics_no_coords.iterrows():
index = row[0]
attrs = row[1].to_dict()
geo_nodes.node[index] = attrs
return geo_nodes
def dataset_store_to_geograph(dataset_store):
"""
convenience function for converting a network stored in a dataset_store
into a GeoGraph
Args:
dataset_store containing a network
Returns:
GeoGraph representation of dataset_store network
TODO: determine projection from dataset_store?
"""
all_nodes = list(dataset_store.cycleNodes()) + \
list(dataset_store.cycleNodes(isFake=True))
np_to_nx_id = {node.id: i for i, node in enumerate(all_nodes)}
coords = [node.getCommonCoordinates() for node in all_nodes]
coords_dict = dict(enumerate(coords))
budget_dict = {i: node.metric for i, node in enumerate(all_nodes)}
G = GeoGraph(coords=coords_dict)
nx.set_node_attributes(G, 'budget', budget_dict)
def seg_to_nx_ids(seg):
"""
Return the networkx segment ids
"""
return (np_to_nx_id[seg.node1_id],
np_to_nx_id[seg.node2_id])
edges = [seg_to_nx_ids(s) for s in
dataset_store.cycleSegments(is_existing=False)]
edge_weights = {seg_to_nx_ids(s): s.weight for s in
dataset_store.cycleSegments(is_existing=False)}
edge_is_existing = {seg_to_nx_ids(s): s.is_existing for s in
dataset_store.cycleSegments(is_existing=False)}
edge_subnet_id = {seg_to_nx_ids(s): s.subnet_id for s in
dataset_store.cycleSegments(is_existing=False)}
G.add_edges_from(edges)
nx.set_edge_attributes(G, 'weight', edge_weights)
nx.set_edge_attributes(G, 'is_existing', edge_is_existing)
nx.set_edge_attributes(G, 'subnet_id', edge_subnet_id)
return G
def test_project_xyz_vs_geo():
"""
ensure that project_onto works the same with xyz vs lat_lon points
"""
net_coords = {'grid-1': [0.0, 0.0],
'grid-2': [10.0, 10.0]}
net_edges = [('grid-1', 'grid-2')]
node_coords = {0: [5.0, 5.0]}
g_net = GeoGraph(gm.PROJ4_LATLONG, net_coords, data=net_edges)
g_nodes = GeoGraph(gm.PROJ4_LATLONG, node_coords)
g_project = g_net.project_onto(g_nodes, spherical_accuracy=True)
g_net_xyz = GeoGraph(gm.PROJ4_LATLONG,
g_net.lon_lat_to_cartesian_coords(),
data=net_edges)
g_nodes_xyz = GeoGraph(gm.PROJ4_LATLONG,
g_nodes.lon_lat_to_cartesian_coords())
g_project_xyz = g_net_xyz.project_onto(g_nodes_xyz, spherical_accuracy=True)
g_project_coords_ll = g_project_xyz.cartesian_to_lon_lat()
def round_coords(coords, round_precision=8):
return {i: tuple(map(lambda c: round(c, round_precision), coord))
for i, coord in coords.items()}
assert(round_coords(g_project_coords_ll) == round_coords(g_project.coords))
def generate_mst(coords):
"""
Generate a min spanning tree based on coordinate
distances
"""
input_proj = gm.PROJ4_LATLONG
if gm.is_in_lon_lat(coords):
input_proj = gm.PROJ4_LATLONG
else:
input_proj = gm.PROJ4_FLAT_EARTH
node_dict = dict(enumerate(coords))
geo_nodes = GeoGraph(input_proj, node_dict)
geo_full = geo_nodes.get_connected_weighted_graph()
geo_mst = nx.minimum_spanning_tree(geo_full)
geo_nodes.add_edges_from(geo_mst.edges(data=True))
return geo_nodes
def test_load_write_json():
"""
ensure that reading/writing js 'node-link' format works
"""
os.mkdir('test/tmp')
node_dict = {0: [0,0], 1: [0,1], 2: [1,0], 3: [1,1]}
g = GeoGraph(gm.PROJ4_LATLONG, node_dict)
g.add_edges_from([(0,1),(1,2),(2,3)])
nio.write_json(g, open('test/tmp/g.js', 'w'))
g2 = nio.load_json(open('test/tmp/g.js', 'r'))
os.remove('test/tmp/g.js')
os.rmdir('test/tmp')
assert nx.is_isomorphic(g, g2,
node_match=operator.eq,
edge_match=operator.eq),\
"expected written and read graphs to match"
def test_load_write_json():
"""
ensure that reading/writing js 'node-link' format works
"""
os.mkdir(os.path.join('test', 'tmp'))
node_dict = {0: [0,0], 1: [0,1], 2: [1,0], 3: [1,1]}
g = GeoGraph(gm.PROJ4_LATLONG, node_dict)
g.add_edges_from([(0,1),(1,2),(2,3)])
json_file_path = os.path.join('test', 'tmp', 'g.json')
nio.write_json(g, open(json_file_path, 'w'))
g2 = nio.read_json_geograph(json_file_path)
os.remove(json_file_path)
os.rmdir(os.path.join('test', 'tmp'))
assert nx.is_isomorphic(g, g2,
node_match=operator.eq,
edge_match=operator.eq),\
"expected written and read graphs to match"
def test_compose():
"""
Ensure that GeoGraph.compose works correctly
geo_left: geo_right (no edges):
0---1 2 0 1
force_disjoint=True:
0---1 2 3 4
force_disjoint=False:
0---1 2
Note that geo_right attributes take precedence over geo_left
when merged
"""
left_coords = [[0.0, 0.0], [0.1, 0.0], [0.2, 0.0]]
left_edges = [(0, 1)]
left_attrs = {0: {'name': 'left0'},
1: {'name': 'left1'},
2: {'name': 'left2'}}
right_coords = [[0.0, 0.0], [0.1, 0.0]]
right_attrs = {0: {'name': 'right0'},
1: {'name': 'right1'}}
geo_left = GeoGraph(coords=dict(enumerate(left_coords)), data=left_edges)
geo_left.node = copy.deepcopy(left_attrs)
geo_right = GeoGraph(coords=dict(enumerate(right_coords)))
geo_right.node = copy.deepcopy(right_attrs)
geo_union = GeoGraph.compose(geo_left, geo_right, force_disjoint=False)
union_attrs = copy.deepcopy(left_attrs)
union_attrs.update(right_attrs)
assert geo_union.nodes() == [0, 1, 2] \
and geo_union.edges() == [(0, 1)] \
and geo_union.node == union_attrs, \
"Non-disjoint composition not correct"
geo_union = GeoGraph.compose(geo_left, geo_right, force_disjoint=True)
assert geo_union.nodes() == [0, 1, 2, 3, 4] \
and geo_union.edges() == [(0, 1)] \
and geo_union.coords == dict(enumerate(left_coords + right_coords)), \
"Disjoint composition not correct"
def grid_and_non_grid():
"""
return networkplan GeoGraph with grid and non-grid components
0 2
| |
+-6-+ 1 3-4-5
(inf)
where node 3 is a fake node connecting node 0 to the grid
"""
node_coords = np.array([[0.0, 1.0], [4.0, 0.0], [4.0, 1.0], [5.0, 0.0],
[6.0, 0.0], [7.0, 0.0], [0.0, 0.0]])
grid = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords)))
budget_values = [5, 5, 5, 5, 5, 5, np.inf]
nx.set_node_attributes(grid, 'budget', dict(enumerate(budget_values)))
grid.add_edges_from([(0, 6), (1, 2), (3, 4), (4, 5)])
return grid
def simple_nodes_disjoint_grid():
"""
return disjoint net plus nodes with fakes
fakes are associated with disjoint subnets
nodes by id (budget in parens)
(5) 0-------1 (5)
| |
+-+-+ +-+-+ <-- disjoint existing grid
Useful for testing treating existing grid as single grid
vs disjoint
"""
# setup grid
grid_coords = np.array([[-1.0, 0.0], [1.0, 0.0], [3.0, 0.0], [5.0, 0.0]])
grid = GeoGraph(gm.PROJ4_FLAT_EARTH,
{'grid-' + str(n): c
for n, c in enumerate(grid_coords)})
nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()})
grid.add_edges_from([('grid-0', 'grid-1'), ('grid-2', 'grid-3')])
# setup input nodes
node_coords = np.array([[0.0, 1.0], [4.0, 1.0]])
nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords)))
budget_values = [5, 5]
nx.set_node_attributes(nodes, 'budget', dict(enumerate(budget_values)))
fakes = [2, 3]
return grid, nodes, fakes
def network_nodes_projections():
"""
Create a network and node graph to be merged, and
the expected result of project_onto for testing
rough picture of this test
4
/
/
/
/
/
/ 2
/ |
/ 6 | 8
/ 5 0---1
3 7
nodes 5,6,7,8 should be projected onto graph path (0,1,2)
graph path (3,4) is meant to test whether a long segment whose
bbox overlaps all nodes interferes with the projection
(it had in the past)
"""
net_coords = [[0.0, 0.0], [3.0, 0.0], [3.0, 3.0],
[-6.0, -1.0], [6.0, 11.0]]
net_edges = [(0, 1), (1, 2), (3, 4)]
node_coords = {5: [-1.0, 0.0], 6: [1.0, 1.0],
7: [4.0, -1.0], 8: [4.0, 1.0]}
projected_coords = {5: [0.0, 0.0], 6: [1.0, 0.0],
7: [3.0, 0.0], 8: [3.0, 1.0]}
g_net = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(net_coords)), data=net_edges)
g_nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, node_coords)
return g_net, g_nodes, projected_coords
def nodes_plus_existing_grid():
"""
return net plus existing grid with certain properties for testing
nodes by id (budget in parens)
1 (3)
\
\
0 (2)
| 2 (5)
| |
+-+-+-+-+-+-+-+-+-+-+ <-- existing grid
"""
# setup grid
grid_coords = np.array([[-5.0, 0.0], [5.0, 0.0]])
grid = GeoGraph(gm.PROJ4_FLAT_EARTH, {'grid-' + str(n): c for n, c in
enumerate(grid_coords)})
nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()})
grid.add_edges_from([('grid-0', 'grid-1')])
# setup input nodes
node_coords = np.array([[0.0, 2.0], [-1.0, 4.0], [4.0, 1.0]])
nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords)))
budget_values = [2, 3, 5]
nx.set_node_attributes(nodes, 'budget', dict(enumerate(budget_values)))
# setup resulting edges when creating msf through the sequence of nodes
# Note: Fake nodes integer label begins at the total number of nodes + 1
# Hence why the fake node in the test is incremented by one on each
# iteration
edges_at_iteration = [[(0, 1)], # 0 connects to fake_node
[(0, 2)], # 0, 1 can't connect
[(0, 3), (2, 5), (1, 0)]] # 2 connects grid
return grid, nodes, edges_at_iteration
def read_geojson_geograph(geojson_path):
"""
Load GeoGraph from geojson (as undirected)
See: docs/geograph_geojson.md for format details
Args:
geojson_path: Path to geojson
Returns:
geograph: GeoGraph object
"""
geojson = json.load(geojson_path)
dict_getter = nested_dict_getter()
# build GeoGraph from a plain-old networkx graph
g = nx.Graph()
# Only add crs if it's explicitly set in crs.properties.name
crs = dict_getter(geojson, ['crs', 'properties', 'name'])
if crs:
try:
prj.Proj(crs)
except Exception as e:
raise SpatialReferenceInvalidException(
"Spatial reference must comply with proj4, got {}" % crs)
# TODO: Apply geojson schema validation
features = geojson['features']
# Currently this separates nodes and coordinates due to structure of
# GeoGraph. This may change to allow Nodes and Edges to have distinct
# Geometry.
nodes, coords = geojson_get_nodes(features)
g.add_nodes_from(nodes)
edges = geojson_get_edges(features)
g.add_edges_from(edges)
if crs is None and gm.is_in_lon_lat(coords.values()):
crs = gm.PROJ4_LATLONG
if crs is None:
warnings.warn(
"Spatial Reference could not be set for {}".format(geojson_path))
# TODO: More srs/projection validation?
return GeoGraph(srs=crs, coords=coords, data=g)
def graph_with_dead_node():
# a 'dead' node is one with insufficient mvMax to connect
# to it's nearest neighbor (c in graph below)
# a(2) - b(10) ---- c(2) ----- c(10)
# 1 4 4
mv_max_values = [2, 10, 2, 10]
coords = np.array([[-1.0, 0.0], [0.0, 0.0], [4.0, 0.0], [8.0, 0.0]])
coords_dict = dict(enumerate(coords))
graph = GeoGraph(coords=coords_dict)
nx.set_node_attributes(graph, 'budget', dict(enumerate(mv_max_values)))
return graph
def graph_high_mvmax_long_edge():
# This graph has an edge between components such that
# if the min FNN is selected incorrectly, it will
# produce a non minimal graph (this is just one case to test)
# All vertices have sufficient MV
# a(10) --- b(10) -------- c(10) --- d(10)
# 3 7 3
mv_max_values = [10, 10, 10, 10]
coords = np.array([[0.0, 0.0], [3.0, 0.0], [10.0, 0.0], [13.0, 0.0]])
coords_dict = dict(enumerate(coords))
graph = GeoGraph(coords=coords_dict)
nx.set_node_attributes(graph, 'budget', dict(enumerate(mv_max_values)))
return graph
def read_shp_geograph(shp, simplify=True):
"""
loads a shapefile into a networkx based GeoGraph object
Args:
shp: string or ogr.DataSource
string path name to shapefile or pre-opened ogr.DataSource
simplify: Only retain start/end nodes of multi-segment lines
Returns:
geograph: GeoGraph
"""
# Note: There's already a nx.read_shp which we've contributed to and
# may want to just adopt over our own read_shp_networkx_graph
g = read_shp_networkx_graph(shp, simplify=simplify, geom_attrs=False)
coords = dict(enumerate(g.nodes()))
# needed for SRS
layer = shp.GetLayer()
spatial_ref = layer.GetSpatialRef()
proj4 = None
if not spatial_ref:
if gm.is_in_lon_lat(coords):
proj4 = gm.PROJ4_LATLONG
else:
warnings.warn("Spatial Reference could not be set for {}".format(
shp.GetName()))
else:
proj4 = spatial_ref.ExportToProj4()
g = nx.convert_node_labels_to_integers(g)
return GeoGraph(srs=proj4, coords=coords, data=g)
def nodes_plus_existing_grid():
"""
return net plus existing grid with certain properties for testing
nodes by id (budget in parens)
1 (3)
\
\
0 (2)
| 2 (5)
| |
+-+-+-+-+-+-+-+-+-+-+ <-- existing grid
"""
# setup grid
grid_coords = np.array([[-5.0, 0.0], [5.0, 0.0]])
grid = GeoGraph(gm.PROJ4_FLAT_EARTH,
{'grid-' + str(n): c
for n, c in enumerate(grid_coords)})
nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()})
grid.add_edges_from([('grid-0', 'grid-1')])
# setup input nodes
node_coords = np.array([[0.0, 2.0], [-1.0, 4.0], [4.0, 1.0]])
nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords)))
budget_values = [2, 3, 5]
nx.set_node_attributes(nodes, 'budget', dict(enumerate(budget_values)))
# setup resulting edges when creating msf through the sequence of nodes
# Note: Fake nodes integer label begins at the total number of nodes + 1
# Hence why the fake node in the test is incremented by one on each
# iteration
edges_at_iteration = [
[(0, 1)], # 0 connects to fake_node
[(0, 2)], # 0, 1 can't connect
[(0, 3), (2, 5), (1, 0)]
] # 2 connects grid
return grid, nodes, edges_at_iteration
def union_reduce(left_geo, right_geo):
return GeoGraph.compose(left_geo, right_geo, args.force_disjoint)
def project_helper(use_rtree, spherical_accuracy):
if use_rtree:
logger.info("building rtree...")
rtree = net.get_rtree_index()
logger.info("projecting nodes...")
return net.project_onto(nodes, rtree_index=rtree,
spherical_accuracy=spherical_accuracy)
else:
logger.info("projecting nodes...")
return net.project_onto(nodes, spherical_accuracy=spherical_accuracy)
# project_onto returns geograph with projected nodes PLUS
# the network edges they were projected onto
projected = project_helper(args.rtree, args.spherical_accuracy)
# get only the projected edges
# construct geograph of only projected edges
projected_edges = GeoGraph(srs=projected.srs)
for node in nodes:
edge = (node, projected.neighbors(node)[0])
projected_edges.add_edge(*edge)
projected_edges.coords[edge[0]] = projected.coords[edge[0]]
projected_edges.coords[edge[1]] = projected.coords[edge[1]]
if(args.write_json):
nio.write_json(projected_edges,
open(os.path.join(args.output_directory, 'projected.json'), 'w'))
else:
nio.write_shp(projected_edges, args.output_directory)
def dataset_store_to_geograph(dataset_store):
"""
convenience function for converting a network stored in a dataset_store
into a GeoGraph
Args:
dataset_store containing a network
Returns:
GeoGraph representation of dataset_store network
TODO: determine projection from dataset_store?
"""
all_nodes = list(dataset_store.cycleNodes()) + \
list(dataset_store.cycleNodes(isFake=True))
# nodes in output GeoGraph are id'd from 0 to n (via enumerate)
np_to_nx_id = {node.id: i for i, node in enumerate(all_nodes)}
coords = [node.getCommonCoordinates() for node in all_nodes]
coords_dict = dict(enumerate(coords))
G = GeoGraph(coords=coords_dict)
# only set population, system and budget for now
# TODO: Do we need all from the output?
for i, node in enumerate(all_nodes):
if not node.is_fake:
properties = {
'budget': node.metric,
'population': node.output['demographics']['population count'],
'system': node.output['metric']['system']
}
G.node[i] = properties
def seg_to_nx_ids(seg):
"""
Return the networkx segment ids
"""
return (np_to_nx_id[seg.node1_id], np_to_nx_id[seg.node2_id])
edges = [
seg_to_nx_ids(s)
for s in dataset_store.cycleSegments(is_existing=False)
]
edge_weights = {
seg_to_nx_ids(s): s.weight
for s in dataset_store.cycleSegments(is_existing=False)
}
edge_is_existing = {
seg_to_nx_ids(s): s.is_existing
for s in dataset_store.cycleSegments(is_existing=False)
}
edge_subnet_id = {
seg_to_nx_ids(s): s.subnet_id
for s in dataset_store.cycleSegments(is_existing=False)
}
G.add_edges_from(edges)
nx.set_edge_attributes(G, 'weight', edge_weights)
nx.set_edge_attributes(G, 'is_existing', edge_is_existing)
nx.set_edge_attributes(G, 'subnet_id', edge_subnet_id)
return G
def union_reduce(left_geo, right_geo):
return GeoGraph.compose(left_geo, right_geo, args.force_disjoint)
def merge_network_and_nodes(network, demand_nodes, single_network=True):
"""
merge the network and nodes GeoGraphs to set up the Graph, UnionFind
(DisjoinSet), and RTree datastructures for use in network algorithms
Args:
network: graph representing existing network
(assumes node ids don't conflict with net (demand) nodes)
demand_nodes: graph of nodes representing demand
single_network: whether subgraphs of network are unioned into
a single network
Returns:
graph: graph with demand nodes and their nearest nodes to the
existing network (i.e. 'fake' nodes)
subgraphs: UnionFind datastructure populated with fake nodes and
associated with the appropriate connected component or the entire
subgraph (depending on ``single_subgraph`` param)
rtree: spatial index populated with the edges from the
existing network
"""
# project demand nodes onto network
rtree = network.get_rtree_index()
grid_with_fakes = network.project_onto(demand_nodes, rtree_index=rtree)
# get only the fake nodes and the associated network edges
demand_node_set = set(demand_nodes.nodes())
net_plus_demand = set(network.nodes()).union(demand_node_set)
fakes = set(grid_with_fakes.nodes()) - net_plus_demand
# fake node should only have 2 neighbors from the existing network
# that is the nearest edge
def get_fake_edge(node): return tuple(set(
grid_with_fakes.neighbors(node)) - demand_node_set)
edge_fakes = [(get_fake_edge(fake), fake) for fake in fakes]
# Init the DisjointSet
subgraphs = UnionFind()
assert len(network.nodes()) > 1, \
"network must have more than 1 node"
if single_network:
# just union all nodes to a single parent
nodes = network.nodes()
# add parent
parent = nodes[0]
subgraphs.add_component(parent, budget=network.node[parent]['budget'])
for node in nodes[1:]:
subgraphs.add_component(node, budget=network.node[node]['budget'])
# The existing grid nodes are on the grid (so distance is 0)
subgraphs.union(parent, node, 0)
else:
# Build the subnet components
# Get the network components to init budget centers
subnets = nx.connected_components(network)
for sub in subnets:
# union all nodes to parent of subnet
parent = sub[0]
subgraphs.add_component(parent,
budget=network.node[parent]['budget'])
# Merge remaining nodes with component
for node in sub[1:]:
subgraphs.add_component(node,
budget=network.node[node]['budget'])
# The existing grid nodes are on the grid (so distance is 0)
subgraphs.union(parent, node, 0)
# setup merged graph to be populated with fake nodes
merged = GeoGraph(demand_nodes.srs, demand_nodes.coords, data=demand_nodes)
# merge fakes in
for ((u, v), fake) in edge_fakes:
# Make sure something wonky isn't going on
assert(subgraphs[u] == subgraphs[v])
# Add the fake node to the big net
merged.add_node(fake, budget=np.inf)
merged.coords[fake] = grid_with_fakes.coords[fake]
# Merge the fake node with the grid subgraph
subgraphs.add_component(fake, budget=np.inf)
subgraphs.union(fake, u, 0)
return merged, subgraphs, rtree
def build_network(demand_nodes,
existing=None,
min_node_count=2,
single_network=True,
network_algorithm='mod_boruvka',
one_based=False
):
"""
project demand nodes onto optional existing supply network and
return the 'optimized' network
Args:
demand_nodes: GeoGraph of demand nodes
existing: GeoGraph of existing grid (assumes node ids
don't conflict with demand_nodes
min_node_count: minimum number of nodes allowed in a subgraph
of the result
network_algorithm: Algorithm from ALGOS to run
one_based: Whether result GeoGraph's nodes should be one_based
(if not, they are 0 based)
Returns:
msf: GeoGraph of minimum spanning forest proposed by the chosen
network algorithm
existing: The existing grid GeoGraph (None if it doesn't exist)
"""
geo_graph = subgraphs = rtree = None
if existing:
log.info("merging network and nodes")
geo_graph, subgraphs, rtree = \
merge_network_and_nodes(existing, demand_nodes,
single_network=single_network)
else:
geo_graph = demand_nodes
log.info("running {} on {} demand nodes and {} total nodes".format(
network_algorithm, len(demand_nodes), len(geo_graph)))
# now run the selected algorithm
network_algo = NetworkerRunner.ALGOS[network_algorithm]
result_geo_graph = network_algo(geo_graph, subgraphs=subgraphs,
rtree=rtree)
# TODO: Remove unreferenced fake nodes?
# now filter out subnetworks via minimum node count
# TODO: update union_all to support GeoGraph?
filtered_graph = nx.union_all(filter(
lambda sub: len(sub.node) >= min_node_count,
nx.connected_component_subgraphs(result_geo_graph)))
# map coords back to geograph
# NOTE: explicit relabel to int as somewhere in filtering above, some
# node ids are set to numpy types which screws up comparisons to tuples
# in write op
# NOTE: relabeling nodes in-place here drops node attributes for some
# reason so create a copy for now
def id_label(i):
id = int(i+1) if one_based else int(i)
return id
msf = None
if filtered_graph:
coords = {id_label(i): result_geo_graph.coords[i]
for i in filtered_graph}
relabeled = nx.relabel_nodes(filtered_graph, {i: id_label(i)
for i in filtered_graph}, copy=True)
msf = GeoGraph(result_geo_graph.srs, coords=coords, data=relabeled)
log.info("filtered result has {} nodes and {} edges".format(
len(msf.nodes()), len(msf.edges())))
return msf
def test_project_xyz_vs_geo():
"""
ensure that project_onto works the same with xyz vs lat_lon points
"""
net_coords = {'grid-1': [0.0, 0.0],
'grid-2': [10.0, 10.0]}
net_edges = [('grid-1', 'grid-2')]
node_coords = {0: [5.0, 5.0]}
g_net = GeoGraph(gm.PROJ4_LATLONG, net_coords, data=net_edges)
g_nodes = GeoGraph(gm.PROJ4_LATLONG, node_coords)
g_project = g_net.project_onto(g_nodes, spherical_accuracy=True)
g_net_xyz = GeoGraph(gm.PROJ4_LATLONG,
g_net.lon_lat_to_cartesian_coords(),
data=net_edges)
g_nodes_xyz = GeoGraph(gm.PROJ4_LATLONG,
g_nodes.lon_lat_to_cartesian_coords())
g_project_xyz = g_net_xyz.project_onto(g_nodes_xyz, spherical_accuracy=True)
g_project_coords_ll = g_project_xyz.cartesian_to_lon_lat()
def round_coords(coords, round_precision=8):
return {i: tuple(map(lambda c: round(c, round_precision), coord))
for i, coord in coords.items()}
assert(round_coords(g_project_coords_ll) == round_coords(g_project.coords))
def merge_network_and_nodes(network, demand_nodes,
single_network=True, spherical_accuracy=False):
"""
merge the network and nodes GeoGraphs to set up the Graph, UnionFind
(DisjoinSet), and RTree datastructures for use in network algorithms
Args:
network: graph representing existing network
(assumes node ids don't conflict with net (demand) nodes)
demand_nodes: graph of nodes representing demand
single_network: whether subgraphs of network are unioned into
a single network
spherical_accuracy: Whether to connect nodes to network on a sphere
Returns:
graph: graph with demand nodes and their nearest nodes to the
existing network (i.e. 'fake' nodes)
subgraphs: UnionFind datastructure populated with fake nodes and
associated with the appropriate connected component or the entire
subgraph (depending on ``single_subgraph`` param)
rtree: spatial index populated with the edges from the
existing network
"""
# project demand nodes onto network
rtree = network.get_rtree_index()
grid_with_fakes = network.project_onto(demand_nodes, rtree_index=rtree,
spherical_accuracy=spherical_accuracy)
# get only the fake nodes and the associated network edges
demand_node_set = set(demand_nodes.nodes())
net_plus_demand = set(network.nodes()).union(demand_node_set)
fakes = set(grid_with_fakes.nodes()) - net_plus_demand
def get_fake_edge(node):
"""
fake node should only have 2 neighbors from the existing network
that is the nearest edge
"""
return tuple(set(grid_with_fakes.neighbors(node)) - demand_node_set)
edge_fakes = [(get_fake_edge(fake), fake) for fake in fakes]
# Init the DisjointSet
subgraphs = UnionFind()
assert len(network.nodes()) > 1, \
"network must have more than 1 node"
if single_network:
# just union all nodes to a single parent
nodes = network.nodes()
# add parent
parent = nodes[0]
subgraphs.add_component(parent, budget=network.node[parent]['budget'])
for node in nodes[1:]:
subgraphs.add_component(node, budget=network.node[node]['budget'])
# The existing grid nodes are on the grid (so distance is 0)
subgraphs.union(parent, node, 0)
else:
# Build the subnet components
# Get the network components to init budget centers
subnets = nx.connected_components(network)
for sub in subnets:
# union all nodes to parent of subnet
sub_list = list(sub)
parent = sub_list[0]
subgraphs.add_component(parent,
budget=network.node[parent]['budget'])
# Merge remaining nodes with component
for node in sub_list[1:]:
subgraphs.add_component(node,
budget=network.node[node]['budget'])
# The existing grid nodes are on the grid (so distance is 0)
subgraphs.union(parent, node, 0)
# setup merged graph to be populated with fake nodes
merged = GeoGraph(demand_nodes.srs, demand_nodes.coords, data=demand_nodes)
# merge fakes in
for ((u, v), fake) in edge_fakes:
# Make sure something wonky isn't going on
assert(subgraphs[u] == subgraphs[v])
# Add the fake node to the big net
# NOTE: fake nodes always have np.inf budget
merged.add_node(fake, budget=np.inf)
merged.coords[fake] = grid_with_fakes.coords[fake]
# Merge the fake node with the grid subgraph
subgraphs.add_component(fake, budget=np.inf)
subgraphs.union(fake, u, 0)
return merged, subgraphs, rtree
def build_network(demand_nodes,
existing=None,
min_node_count=2,
single_network=True,
network_algorithm='mod_boruvka',
spherical_accuracy=False,
one_based=False):
"""
project demand nodes onto optional existing supply network and
return the 'optimized' network
Args:
demand_nodes: GeoGraph of demand nodes
existing: GeoGraph of existing grid (assumes node ids
don't conflict with demand_nodes
min_node_count: minimum number of nodes allowed in a subgraph
of the result
network_algorithm: Algorithm from ALGOS to run
spherical_accuracy: Whether to connect nodes to network on a sphere
one_based: Whether result GeoGraph's nodes should be one_based
(if not, they are 0 based)
Returns:
msf: GeoGraph of minimum spanning forest proposed by the chosen
network algorithm
existing: The existing grid GeoGraph (None if it doesn't exist)
"""
geo_graph = subgraphs = rtree = None
if existing:
log.info("merging network and nodes")
geo_graph, subgraphs, rtree = \
merge_network_and_nodes(existing, demand_nodes,
single_network=single_network,
spherical_accuracy=spherical_accuracy)
else:
geo_graph = demand_nodes
log.info("running {} on {} demand nodes and {} total nodes".format(
network_algorithm, len(demand_nodes), len(geo_graph)))
# now run the selected algorithm
network_algo = NetworkerRunner.ALGOS[network_algorithm]
result_geo_graph = network_algo(geo_graph, subgraphs=subgraphs,
rtree=rtree)
filtered_graph = filter_min_node_subnetworks(result_geo_graph,
min_node_count)
# map coords back to geograph
# NOTE: explicit relabel to int as somewhere in filtering above, some
# node ids are set to numpy types which screws up comparisons to tuples
# in write op
# NOTE: relabeling nodes in-place here drops node attributes for some
# reason so create a copy for now
def id_label(i):
id = int(i+1) if one_based else int(i)
return id
msf = GeoGraph(result_geo_graph.srs)
if filtered_graph:
coords = {id_label(i): result_geo_graph.coords[i]
for i in filtered_graph}
relabeled = nx.relabel_nodes(filtered_graph, {i: id_label(i)
for i in filtered_graph},
copy=True)
msf = GeoGraph(result_geo_graph.srs, coords=coords, data=relabeled)
log.info("filtered result has {} nodes and {} edges".format(
len(msf.nodes()), len(msf.edges())))
return msf
def nodes_plus_grid():
"""
Return: nodes as graph and grid as UnionFind/Rtree combo
This example input demonstrates the "more" optimal nature
of mod_boruvka vs mod_kruskal.
2(10)
|
1(4) |
sqrt(5){ /| |
/ | |
/ | }3 | } 5
(2)0 | |
1{| | |
+-+-3-+-4-+-+-+-+-+-+-+5-+-+-+ <-- existing grid
In this case, all nodes will be connected via either algorithm,
but the graph produced by mod_kruskal will have edge (4,1) whereas
mod_boruvka will produce a graph with edge (0,1).
Therefore, the mod_boruvka graph is more optimal.
"""
mv_max_values = [2, 4, 10]
coords = np.array([[0.0, 1.0], [1.0, 3.0], [10.0, 5.0]])
coords_dict = dict(enumerate(coords))
nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, coords=coords_dict)
nx.set_node_attributes(nodes, 'budget', dict(enumerate(mv_max_values)))
grid_coords = np.array([[-5.0, 0.0], [15.0, 0.0]])
grid = GeoGraph(gm.PROJ4_FLAT_EARTH,
{'grid-' + str(n): c
for n, c in enumerate(grid_coords)})
nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()})
grid.add_edges_from([('grid-0', 'grid-1')])
# now find projections onto grid
rtree = grid.get_rtree_index()
projected = grid.project_onto(nodes, rtree_index=rtree)
projected.remove_nodes_from(grid)
projected.remove_nodes_from(nodes)
# populate disjoint set of subgraphs
subgraphs = UnionFind()
# only one connected component, so just associate all nodes
# with first node of grid
parent = grid.nodes()[0]
subgraphs.add_component(parent, budget=grid.node[parent]['budget'])
for node in grid.nodes()[1:]:
subgraphs.add_component(node, budget=grid.node[node]['budget'])
subgraphs.union(parent, node, 0)
# and the projected "fake" nodes
for node in projected.nodes():
subgraphs.add_component(node, budget=np.inf)
subgraphs.union(parent, node, 0)
# add projected nodes to node set
nodes.add_nodes_from(projected, budget=np.inf)
# merge coords
nodes.coords = dict(nodes.coords, **projected.coords)
return nodes, subgraphs, rtree
def nodes_plus_grid():
"""
Return: nodes as graph and grid as UnionFind/Rtree combo
This example input demonstrates the "more" optimal nature
of mod_boruvka vs mod_kruskal.
2(10)
|
1(4) |
sqrt(5){ /| |
/ | |
/ | }3 | } 5
(2)0 | |
1{| | |
+-+-3-+-4-+-+-+-+-+-+-+5-+-+-+ <-- existing grid
In this case, all nodes will be connected via either algorithm,
but the graph produced by mod_kruskal will have edge (4,1) whereas
mod_boruvka will produce a graph with edge (0,1).
Therefore, the mod_boruvka graph is more optimal.
"""
mv_max_values = [2, 4, 10]
coords = np.array([[0.0, 1.0], [1.0, 3.0], [10.0, 5.0]])
coords_dict = dict(enumerate(coords))
nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, coords=coords_dict)
nx.set_node_attributes(nodes, 'budget', dict(enumerate(mv_max_values)))
grid_coords = np.array([[-5.0, 0.0], [15.0, 0.0]])
grid = GeoGraph(gm.PROJ4_FLAT_EARTH, {'grid-' + str(n): c for n, c in
enumerate(grid_coords)})
nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()})
grid.add_edges_from([('grid-0', 'grid-1')])
# now find projections onto grid
rtree = grid.get_rtree_index()
projected = grid.project_onto(nodes, rtree_index=rtree)
projected.remove_nodes_from(grid)
projected.remove_nodes_from(nodes)
# populate disjoint set of subgraphs
subgraphs = UnionFind()
# only one connected component, so just associate all nodes
# with first node of grid
parent = grid.nodes()[0]
subgraphs.add_component(parent, budget=grid.node[parent]['budget'])
for node in grid.nodes()[1:]:
subgraphs.add_component(node, budget=grid.node[node]['budget'])
subgraphs.union(parent, node, 0)
# and the projected "fake" nodes
for node in projected.nodes():
subgraphs.add_component(node, budget=np.inf)
subgraphs.union(parent, node, 0)
# add projected nodes to node set
nodes.add_nodes_from(projected, budget=np.inf)
# merge coords
nodes.coords = dict(nodes.coords, **projected.coords)
return nodes, subgraphs, rtree