def test_merge_nearby_nodes(): """ --2 2 / / -- / / / 1 / \ / 0-----------5 => 0------------5 / 3 | 4 """ coords = [[ 0, 0], [-1, 1], [ 2, 6], [-1, -1], [-1, -2], [ 5, 0]] edges = [(0, 1), (1, 2), (0, 3), (3, 4), (0, 5)] geo = GeoGraph(coords=dict(enumerate(coords)), data=edges) geo.merge_nearby_nodes(radius=2.0) assert geo.edges() == [(0, 2), (0, 5)],\ "nodes were not merged correctly"
def simple_nodes_disjoint_grid(): """ return disjoint net plus nodes with fakes fakes are associated with disjoint subnets nodes by id (budget in parens) (5) 0-------1 (5) | | +-+-+ +-+-+ <-- disjoint existing grid Useful for testing treating existing grid as single grid vs disjoint """ # setup grid grid_coords = np.array([[-1.0, 0.0], [1.0, 0.0], [3.0, 0.0], [5.0, 0.0]]) grid = GeoGraph(gm.PROJ4_FLAT_EARTH, {'grid-' + str(n): c for n, c in enumerate(grid_coords)}) nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()}) grid.add_edges_from([('grid-0', 'grid-1'), ('grid-2', 'grid-3')]) # setup input nodes node_coords = np.array([[0.0, 1.0], [4.0, 1.0]]) nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords))) budget_values = [5, 5] nx.set_node_attributes(nodes, 'budget', dict(enumerate(budget_values))) fakes = [2, 3] return grid, nodes, fakes
def grid_and_non_grid(): """ return networkplan GeoGraph with grid and non-grid components 0 2 | | +-6-+ 1 3-4-5 (inf) where node 3 is a fake node connecting node 0 to the grid """ node_coords = np.array([[0.0, 1.0], [4.0, 0.0], [4.0, 1.0], [5.0, 0.0], [6.0, 0.0], [7.0, 0.0], [0.0, 0.0]]) grid = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords))) budget_values = [5, 5, 5, 5, 5, 5, np.inf] nx.set_node_attributes(grid, 'budget', dict(enumerate(budget_values))) grid.add_edges_from([(0, 6), (1, 2), (3, 4), (4, 5)]) return grid
def dataset_store_to_geograph(dataset_store): """ convenience function for converting a network stored in a dataset_store into a GeoGraph Args: dataset_store containing a network Returns: GeoGraph representation of dataset_store network TODO: determine projection from dataset_store? """ all_nodes = list(dataset_store.cycleNodes()) + \ list(dataset_store.cycleNodes(isFake=True)) # nodes in output GeoGraph are id'd from 0 to n (via enumerate) np_to_nx_id = {node.id: i for i, node in enumerate(all_nodes)} coords = [node.getCommonCoordinates() for node in all_nodes] coords_dict = dict(enumerate(coords)) G = GeoGraph(coords=coords_dict) # only set population, system and budget for now # TODO: Do we need all from the output? for i, node in enumerate(all_nodes): if not node.is_fake: properties = { 'budget': node.metric, 'population': node.output['demographics']['population count'], 'system': node.output['metric']['system'] } G.node[i] = properties def seg_to_nx_ids(seg): """ Return the networkx segment ids """ return (np_to_nx_id[seg.node1_id], np_to_nx_id[seg.node2_id]) edges = [seg_to_nx_ids(s) for s in dataset_store.cycleSegments(is_existing=False)] edge_weights = {seg_to_nx_ids(s): s.weight for s in dataset_store.cycleSegments(is_existing=False)} edge_is_existing = {seg_to_nx_ids(s): s.is_existing for s in dataset_store.cycleSegments(is_existing=False)} edge_subnet_id = {seg_to_nx_ids(s): s.subnet_id for s in dataset_store.cycleSegments(is_existing=False)} G.add_edges_from(edges) nx.set_edge_attributes(G, 'weight', edge_weights) nx.set_edge_attributes(G, 'is_existing', edge_is_existing) nx.set_edge_attributes(G, 'subnet_id', edge_subnet_id) return G
def read_csv_geograph(csv_file, x_column="X", y_column="Y"): """ load nodes csv into GeoGraph (nodes only for now) Args: csv_file: nodal metrics csv file as string or file x_column, y_column: col names to take x, y from Returns: GeoGraph of nodes including all attributes from input csv """ input_proj = read_csv_projection(csv_file) header_row = 1 if input_proj else 0 # read in the csv # NOTE: Convert x,y via float cast to preserve precision of input # go back to beginning 1st csv_file.seek(0) metrics = pd.read_csv(csv_file, header=header_row, converters={ x_column: float, y_column: float }) coord_cols = [x_column, y_column] assert all([hasattr(metrics, col) for col in coord_cols]), \ "metrics file does not contain coordinate columns {}, {}".\ format(x_column, y_column) # Stack the coords coords = np.column_stack(map(metrics.get, coord_cols)) # set default projection if not input_proj: if gm.is_in_lon_lat(coords): input_proj = gm.PROJ4_LATLONG else: input_proj = gm.PROJ4_FLAT_EARTH coords_dict = dict(enumerate(coords)) geo_nodes = GeoGraph(input_proj, coords_dict) # populate the rest of the attributes metrics_no_coords = metrics[metrics.columns.difference(coord_cols)] for row in metrics_no_coords.iterrows(): index = row[0] attrs = row[1].to_dict() geo_nodes.node[index] = attrs return geo_nodes
def _get_demand_nodes(self, input_proj=None): """ Converts the dataset_store metrics records to a GeoGraph of nodes (prereq: _run_metric_model to populate store) Args: input_proj: projection of demand node coordinates Returns: GeoGraph: demand nodes as GeoGraph """ coords = [ node.getCommonCoordinates() for node in self.store.cycleNodes() ] # set default projection if not input_proj: input_proj = self._get_default_proj4(coords) # NOTE: Although dataset_store nodes id sequence starts at 1 # leave the GeoGraph ids 0 based because there are places in the # network algorithm that assume 0 based coords # This will be realigned later coords_dict = {i: coord for i, coord in enumerate(coords)} budget_dict = { i: node.metric for i, node in enumerate(self.store.cycleNodes()) } geo_nodes = GeoGraph(input_proj, coords_dict) nx.set_node_attributes(geo_nodes, 'budget', budget_dict) return geo_nodes
def random_settlements(n): coords = np.random.uniform(size=(n, 2)) # get all perm's of points (repetitions are ok here) points_left = np.tile(coords, (len(coords), 1)) points_right = np.repeat(coords, len(coords), axis=0) point_pairs = np.concatenate( (points_left[:, np.newaxis], points_right[:, np.newaxis]), axis=1) all_dists = gm.spherical_distance_haversine(point_pairs) full_dist_matrix = all_dists.reshape(len(coords), len(coords)) zero_indices = (np.array(range(len(coords))) * (len(coords) + 1)) non_zero_dists = np.delete(all_dists, zero_indices).\ reshape((len(coords), len(coords) - 1)) # find all minimum distances # apply min over ranges of the dist array min_dists = np.min(non_zero_dists, axis=1) # assign same median budget to all nodes # outside a really degenerate case (all edges in line in shortest # distance order...) # this should ensure some "dead" nodes budget_vals = np.repeat(np.median(min_dists), len(coords)) # build graph graph = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(coords))) nx.set_node_attributes(graph, 'budget', dict(enumerate(budget_vals))) return graph, full_dist_matrix
def read_json_geograph(json_file): """ Args: json_file: path to json file as string or file Assumes the json is in networkx link-node format """ js = json.load(json_file) g = json_graph.node_link_graph(js) assert all([nd.has_key('coords') for nd in g.node.values()]),\ "json node-link graph must have nodes with coords for GeoGraph" # get coords coords = [v['coords'] for v in g.node.values()] # set default projection input_proj = "" if gm.is_in_lon_lat(coords): input_proj = gm.PROJ4_LATLONG else: input_proj = gm.PROJ4_FLAT_EARTH coords_dict = {k: v['coords'] for k, v in g.node.items()} # now get rid of 'coords' key,val for each node for node in g.node.values(): node.pop('coords', None) geo_nodes = GeoGraph(srs=input_proj, coords=coords_dict, data=g) return geo_nodes
def load_nodes(filename="metrics.csv", x_column="X", y_column="Y"): """ load nodes csv into GeoGraph (nodes only) Args: filename: nodal metrics csv file x_column, y_column: col names to take x, y from Returns: GeoGraph of nodes including all attributes from input csv """ input_proj = csv_projection(filename) header_row = 1 if input_proj else 0 # read in the csv # NOTE: Convert x,y via float cast to preserve precision of input metrics = pd.read_csv(filename, header=header_row, converters={x_column: float, y_column: float}) coord_cols = [x_column, y_column] assert all([hasattr(metrics, col) for col in coord_cols]), \ "metrics file does not contain coordinate columns {}, {}".\ format(x_column, y_column) # Stack the coords coords = np.column_stack(map(metrics.get, coord_cols)) # set default projection if not input_proj: if gm.is_in_lon_lat(coords): input_proj = gm.PROJ4_LATLONG else: input_proj = gm.PROJ4_FLAT_EARTH coords_dict = dict(enumerate(coords)) geo_nodes = GeoGraph(input_proj, coords_dict) # populate the rest of the attributes metrics_no_coords = metrics[metrics.columns.difference(coord_cols)] for row in metrics_no_coords.iterrows(): index = row[0] attrs = row[1].to_dict() geo_nodes.node[index] = attrs return geo_nodes
def dataset_store_to_geograph(dataset_store): """ convenience function for converting a network stored in a dataset_store into a GeoGraph Args: dataset_store containing a network Returns: GeoGraph representation of dataset_store network TODO: determine projection from dataset_store? """ all_nodes = list(dataset_store.cycleNodes()) + \ list(dataset_store.cycleNodes(isFake=True)) np_to_nx_id = {node.id: i for i, node in enumerate(all_nodes)} coords = [node.getCommonCoordinates() for node in all_nodes] coords_dict = dict(enumerate(coords)) budget_dict = {i: node.metric for i, node in enumerate(all_nodes)} G = GeoGraph(coords=coords_dict) nx.set_node_attributes(G, 'budget', budget_dict) def seg_to_nx_ids(seg): """ Return the networkx segment ids """ return (np_to_nx_id[seg.node1_id], np_to_nx_id[seg.node2_id]) edges = [seg_to_nx_ids(s) for s in dataset_store.cycleSegments(is_existing=False)] edge_weights = {seg_to_nx_ids(s): s.weight for s in dataset_store.cycleSegments(is_existing=False)} edge_is_existing = {seg_to_nx_ids(s): s.is_existing for s in dataset_store.cycleSegments(is_existing=False)} edge_subnet_id = {seg_to_nx_ids(s): s.subnet_id for s in dataset_store.cycleSegments(is_existing=False)} G.add_edges_from(edges) nx.set_edge_attributes(G, 'weight', edge_weights) nx.set_edge_attributes(G, 'is_existing', edge_is_existing) nx.set_edge_attributes(G, 'subnet_id', edge_subnet_id) return G
def test_project_xyz_vs_geo(): """ ensure that project_onto works the same with xyz vs lat_lon points """ net_coords = {'grid-1': [0.0, 0.0], 'grid-2': [10.0, 10.0]} net_edges = [('grid-1', 'grid-2')] node_coords = {0: [5.0, 5.0]} g_net = GeoGraph(gm.PROJ4_LATLONG, net_coords, data=net_edges) g_nodes = GeoGraph(gm.PROJ4_LATLONG, node_coords) g_project = g_net.project_onto(g_nodes, spherical_accuracy=True) g_net_xyz = GeoGraph(gm.PROJ4_LATLONG, g_net.lon_lat_to_cartesian_coords(), data=net_edges) g_nodes_xyz = GeoGraph(gm.PROJ4_LATLONG, g_nodes.lon_lat_to_cartesian_coords()) g_project_xyz = g_net_xyz.project_onto(g_nodes_xyz, spherical_accuracy=True) g_project_coords_ll = g_project_xyz.cartesian_to_lon_lat() def round_coords(coords, round_precision=8): return {i: tuple(map(lambda c: round(c, round_precision), coord)) for i, coord in coords.items()} assert(round_coords(g_project_coords_ll) == round_coords(g_project.coords))
def generate_mst(coords): """ Generate a min spanning tree based on coordinate distances """ input_proj = gm.PROJ4_LATLONG if gm.is_in_lon_lat(coords): input_proj = gm.PROJ4_LATLONG else: input_proj = gm.PROJ4_FLAT_EARTH node_dict = dict(enumerate(coords)) geo_nodes = GeoGraph(input_proj, node_dict) geo_full = geo_nodes.get_connected_weighted_graph() geo_mst = nx.minimum_spanning_tree(geo_full) geo_nodes.add_edges_from(geo_mst.edges(data=True)) return geo_nodes
def test_load_write_json(): """ ensure that reading/writing js 'node-link' format works """ os.mkdir('test/tmp') node_dict = {0: [0,0], 1: [0,1], 2: [1,0], 3: [1,1]} g = GeoGraph(gm.PROJ4_LATLONG, node_dict) g.add_edges_from([(0,1),(1,2),(2,3)]) nio.write_json(g, open('test/tmp/g.js', 'w')) g2 = nio.load_json(open('test/tmp/g.js', 'r')) os.remove('test/tmp/g.js') os.rmdir('test/tmp') assert nx.is_isomorphic(g, g2, node_match=operator.eq, edge_match=operator.eq),\ "expected written and read graphs to match"
def test_load_write_json(): """ ensure that reading/writing js 'node-link' format works """ os.mkdir(os.path.join('test', 'tmp')) node_dict = {0: [0,0], 1: [0,1], 2: [1,0], 3: [1,1]} g = GeoGraph(gm.PROJ4_LATLONG, node_dict) g.add_edges_from([(0,1),(1,2),(2,3)]) json_file_path = os.path.join('test', 'tmp', 'g.json') nio.write_json(g, open(json_file_path, 'w')) g2 = nio.read_json_geograph(json_file_path) os.remove(json_file_path) os.rmdir(os.path.join('test', 'tmp')) assert nx.is_isomorphic(g, g2, node_match=operator.eq, edge_match=operator.eq),\ "expected written and read graphs to match"
def test_compose(): """ Ensure that GeoGraph.compose works correctly geo_left: geo_right (no edges): 0---1 2 0 1 force_disjoint=True: 0---1 2 3 4 force_disjoint=False: 0---1 2 Note that geo_right attributes take precedence over geo_left when merged """ left_coords = [[0.0, 0.0], [0.1, 0.0], [0.2, 0.0]] left_edges = [(0, 1)] left_attrs = {0: {'name': 'left0'}, 1: {'name': 'left1'}, 2: {'name': 'left2'}} right_coords = [[0.0, 0.0], [0.1, 0.0]] right_attrs = {0: {'name': 'right0'}, 1: {'name': 'right1'}} geo_left = GeoGraph(coords=dict(enumerate(left_coords)), data=left_edges) geo_left.node = copy.deepcopy(left_attrs) geo_right = GeoGraph(coords=dict(enumerate(right_coords))) geo_right.node = copy.deepcopy(right_attrs) geo_union = GeoGraph.compose(geo_left, geo_right, force_disjoint=False) union_attrs = copy.deepcopy(left_attrs) union_attrs.update(right_attrs) assert geo_union.nodes() == [0, 1, 2] \ and geo_union.edges() == [(0, 1)] \ and geo_union.node == union_attrs, \ "Non-disjoint composition not correct" geo_union = GeoGraph.compose(geo_left, geo_right, force_disjoint=True) assert geo_union.nodes() == [0, 1, 2, 3, 4] \ and geo_union.edges() == [(0, 1)] \ and geo_union.coords == dict(enumerate(left_coords + right_coords)), \ "Disjoint composition not correct"
def grid_and_non_grid(): """ return networkplan GeoGraph with grid and non-grid components 0 2 | | +-6-+ 1 3-4-5 (inf) where node 3 is a fake node connecting node 0 to the grid """ node_coords = np.array([[0.0, 1.0], [4.0, 0.0], [4.0, 1.0], [5.0, 0.0], [6.0, 0.0], [7.0, 0.0], [0.0, 0.0]]) grid = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords))) budget_values = [5, 5, 5, 5, 5, 5, np.inf] nx.set_node_attributes(grid, 'budget', dict(enumerate(budget_values))) grid.add_edges_from([(0, 6), (1, 2), (3, 4), (4, 5)]) return grid
def simple_nodes_disjoint_grid(): """ return disjoint net plus nodes with fakes fakes are associated with disjoint subnets nodes by id (budget in parens) (5) 0-------1 (5) | | +-+-+ +-+-+ <-- disjoint existing grid Useful for testing treating existing grid as single grid vs disjoint """ # setup grid grid_coords = np.array([[-1.0, 0.0], [1.0, 0.0], [3.0, 0.0], [5.0, 0.0]]) grid = GeoGraph(gm.PROJ4_FLAT_EARTH, {'grid-' + str(n): c for n, c in enumerate(grid_coords)}) nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()}) grid.add_edges_from([('grid-0', 'grid-1'), ('grid-2', 'grid-3')]) # setup input nodes node_coords = np.array([[0.0, 1.0], [4.0, 1.0]]) nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords))) budget_values = [5, 5] nx.set_node_attributes(nodes, 'budget', dict(enumerate(budget_values))) fakes = [2, 3] return grid, nodes, fakes
def network_nodes_projections(): """ Create a network and node graph to be merged, and the expected result of project_onto for testing rough picture of this test 4 / / / / / / 2 / | / 6 | 8 / 5 0---1 3 7 nodes 5,6,7,8 should be projected onto graph path (0,1,2) graph path (3,4) is meant to test whether a long segment whose bbox overlaps all nodes interferes with the projection (it had in the past) """ net_coords = [[0.0, 0.0], [3.0, 0.0], [3.0, 3.0], [-6.0, -1.0], [6.0, 11.0]] net_edges = [(0, 1), (1, 2), (3, 4)] node_coords = {5: [-1.0, 0.0], 6: [1.0, 1.0], 7: [4.0, -1.0], 8: [4.0, 1.0]} projected_coords = {5: [0.0, 0.0], 6: [1.0, 0.0], 7: [3.0, 0.0], 8: [3.0, 1.0]} g_net = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(net_coords)), data=net_edges) g_nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, node_coords) return g_net, g_nodes, projected_coords
def nodes_plus_existing_grid(): """ return net plus existing grid with certain properties for testing nodes by id (budget in parens) 1 (3) \ \ 0 (2) | 2 (5) | | +-+-+-+-+-+-+-+-+-+-+ <-- existing grid """ # setup grid grid_coords = np.array([[-5.0, 0.0], [5.0, 0.0]]) grid = GeoGraph(gm.PROJ4_FLAT_EARTH, {'grid-' + str(n): c for n, c in enumerate(grid_coords)}) nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()}) grid.add_edges_from([('grid-0', 'grid-1')]) # setup input nodes node_coords = np.array([[0.0, 2.0], [-1.0, 4.0], [4.0, 1.0]]) nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords))) budget_values = [2, 3, 5] nx.set_node_attributes(nodes, 'budget', dict(enumerate(budget_values))) # setup resulting edges when creating msf through the sequence of nodes # Note: Fake nodes integer label begins at the total number of nodes + 1 # Hence why the fake node in the test is incremented by one on each # iteration edges_at_iteration = [[(0, 1)], # 0 connects to fake_node [(0, 2)], # 0, 1 can't connect [(0, 3), (2, 5), (1, 0)]] # 2 connects grid return grid, nodes, edges_at_iteration
def read_geojson_geograph(geojson_path): """ Load GeoGraph from geojson (as undirected) See: docs/geograph_geojson.md for format details Args: geojson_path: Path to geojson Returns: geograph: GeoGraph object """ geojson = json.load(geojson_path) dict_getter = nested_dict_getter() # build GeoGraph from a plain-old networkx graph g = nx.Graph() # Only add crs if it's explicitly set in crs.properties.name crs = dict_getter(geojson, ['crs', 'properties', 'name']) if crs: try: prj.Proj(crs) except Exception as e: raise SpatialReferenceInvalidException( "Spatial reference must comply with proj4, got {}" % crs) # TODO: Apply geojson schema validation features = geojson['features'] # Currently this separates nodes and coordinates due to structure of # GeoGraph. This may change to allow Nodes and Edges to have distinct # Geometry. nodes, coords = geojson_get_nodes(features) g.add_nodes_from(nodes) edges = geojson_get_edges(features) g.add_edges_from(edges) if crs is None and gm.is_in_lon_lat(coords.values()): crs = gm.PROJ4_LATLONG if crs is None: warnings.warn( "Spatial Reference could not be set for {}".format(geojson_path)) # TODO: More srs/projection validation? return GeoGraph(srs=crs, coords=coords, data=g)
def graph_with_dead_node(): # a 'dead' node is one with insufficient mvMax to connect # to it's nearest neighbor (c in graph below) # a(2) - b(10) ---- c(2) ----- c(10) # 1 4 4 mv_max_values = [2, 10, 2, 10] coords = np.array([[-1.0, 0.0], [0.0, 0.0], [4.0, 0.0], [8.0, 0.0]]) coords_dict = dict(enumerate(coords)) graph = GeoGraph(coords=coords_dict) nx.set_node_attributes(graph, 'budget', dict(enumerate(mv_max_values))) return graph
def graph_high_mvmax_long_edge(): # This graph has an edge between components such that # if the min FNN is selected incorrectly, it will # produce a non minimal graph (this is just one case to test) # All vertices have sufficient MV # a(10) --- b(10) -------- c(10) --- d(10) # 3 7 3 mv_max_values = [10, 10, 10, 10] coords = np.array([[0.0, 0.0], [3.0, 0.0], [10.0, 0.0], [13.0, 0.0]]) coords_dict = dict(enumerate(coords)) graph = GeoGraph(coords=coords_dict) nx.set_node_attributes(graph, 'budget', dict(enumerate(mv_max_values))) return graph
def read_shp_geograph(shp, simplify=True): """ loads a shapefile into a networkx based GeoGraph object Args: shp: string or ogr.DataSource string path name to shapefile or pre-opened ogr.DataSource simplify: Only retain start/end nodes of multi-segment lines Returns: geograph: GeoGraph """ # Note: There's already a nx.read_shp which we've contributed to and # may want to just adopt over our own read_shp_networkx_graph g = read_shp_networkx_graph(shp, simplify=simplify, geom_attrs=False) coords = dict(enumerate(g.nodes())) # needed for SRS layer = shp.GetLayer() spatial_ref = layer.GetSpatialRef() proj4 = None if not spatial_ref: if gm.is_in_lon_lat(coords): proj4 = gm.PROJ4_LATLONG else: warnings.warn("Spatial Reference could not be set for {}".format( shp.GetName())) else: proj4 = spatial_ref.ExportToProj4() g = nx.convert_node_labels_to_integers(g) return GeoGraph(srs=proj4, coords=coords, data=g)
def nodes_plus_existing_grid(): """ return net plus existing grid with certain properties for testing nodes by id (budget in parens) 1 (3) \ \ 0 (2) | 2 (5) | | +-+-+-+-+-+-+-+-+-+-+ <-- existing grid """ # setup grid grid_coords = np.array([[-5.0, 0.0], [5.0, 0.0]]) grid = GeoGraph(gm.PROJ4_FLAT_EARTH, {'grid-' + str(n): c for n, c in enumerate(grid_coords)}) nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()}) grid.add_edges_from([('grid-0', 'grid-1')]) # setup input nodes node_coords = np.array([[0.0, 2.0], [-1.0, 4.0], [4.0, 1.0]]) nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, dict(enumerate(node_coords))) budget_values = [2, 3, 5] nx.set_node_attributes(nodes, 'budget', dict(enumerate(budget_values))) # setup resulting edges when creating msf through the sequence of nodes # Note: Fake nodes integer label begins at the total number of nodes + 1 # Hence why the fake node in the test is incremented by one on each # iteration edges_at_iteration = [ [(0, 1)], # 0 connects to fake_node [(0, 2)], # 0, 1 can't connect [(0, 3), (2, 5), (1, 0)] ] # 2 connects grid return grid, nodes, edges_at_iteration
def union_reduce(left_geo, right_geo): return GeoGraph.compose(left_geo, right_geo, args.force_disjoint)
def project_helper(use_rtree, spherical_accuracy): if use_rtree: logger.info("building rtree...") rtree = net.get_rtree_index() logger.info("projecting nodes...") return net.project_onto(nodes, rtree_index=rtree, spherical_accuracy=spherical_accuracy) else: logger.info("projecting nodes...") return net.project_onto(nodes, spherical_accuracy=spherical_accuracy) # project_onto returns geograph with projected nodes PLUS # the network edges they were projected onto projected = project_helper(args.rtree, args.spherical_accuracy) # get only the projected edges # construct geograph of only projected edges projected_edges = GeoGraph(srs=projected.srs) for node in nodes: edge = (node, projected.neighbors(node)[0]) projected_edges.add_edge(*edge) projected_edges.coords[edge[0]] = projected.coords[edge[0]] projected_edges.coords[edge[1]] = projected.coords[edge[1]] if(args.write_json): nio.write_json(projected_edges, open(os.path.join(args.output_directory, 'projected.json'), 'w')) else: nio.write_shp(projected_edges, args.output_directory)
def dataset_store_to_geograph(dataset_store): """ convenience function for converting a network stored in a dataset_store into a GeoGraph Args: dataset_store containing a network Returns: GeoGraph representation of dataset_store network TODO: determine projection from dataset_store? """ all_nodes = list(dataset_store.cycleNodes()) + \ list(dataset_store.cycleNodes(isFake=True)) # nodes in output GeoGraph are id'd from 0 to n (via enumerate) np_to_nx_id = {node.id: i for i, node in enumerate(all_nodes)} coords = [node.getCommonCoordinates() for node in all_nodes] coords_dict = dict(enumerate(coords)) G = GeoGraph(coords=coords_dict) # only set population, system and budget for now # TODO: Do we need all from the output? for i, node in enumerate(all_nodes): if not node.is_fake: properties = { 'budget': node.metric, 'population': node.output['demographics']['population count'], 'system': node.output['metric']['system'] } G.node[i] = properties def seg_to_nx_ids(seg): """ Return the networkx segment ids """ return (np_to_nx_id[seg.node1_id], np_to_nx_id[seg.node2_id]) edges = [ seg_to_nx_ids(s) for s in dataset_store.cycleSegments(is_existing=False) ] edge_weights = { seg_to_nx_ids(s): s.weight for s in dataset_store.cycleSegments(is_existing=False) } edge_is_existing = { seg_to_nx_ids(s): s.is_existing for s in dataset_store.cycleSegments(is_existing=False) } edge_subnet_id = { seg_to_nx_ids(s): s.subnet_id for s in dataset_store.cycleSegments(is_existing=False) } G.add_edges_from(edges) nx.set_edge_attributes(G, 'weight', edge_weights) nx.set_edge_attributes(G, 'is_existing', edge_is_existing) nx.set_edge_attributes(G, 'subnet_id', edge_subnet_id) return G
def union_reduce(left_geo, right_geo): return GeoGraph.compose(left_geo, right_geo, args.force_disjoint)
def merge_network_and_nodes(network, demand_nodes, single_network=True): """ merge the network and nodes GeoGraphs to set up the Graph, UnionFind (DisjoinSet), and RTree datastructures for use in network algorithms Args: network: graph representing existing network (assumes node ids don't conflict with net (demand) nodes) demand_nodes: graph of nodes representing demand single_network: whether subgraphs of network are unioned into a single network Returns: graph: graph with demand nodes and their nearest nodes to the existing network (i.e. 'fake' nodes) subgraphs: UnionFind datastructure populated with fake nodes and associated with the appropriate connected component or the entire subgraph (depending on ``single_subgraph`` param) rtree: spatial index populated with the edges from the existing network """ # project demand nodes onto network rtree = network.get_rtree_index() grid_with_fakes = network.project_onto(demand_nodes, rtree_index=rtree) # get only the fake nodes and the associated network edges demand_node_set = set(demand_nodes.nodes()) net_plus_demand = set(network.nodes()).union(demand_node_set) fakes = set(grid_with_fakes.nodes()) - net_plus_demand # fake node should only have 2 neighbors from the existing network # that is the nearest edge def get_fake_edge(node): return tuple(set( grid_with_fakes.neighbors(node)) - demand_node_set) edge_fakes = [(get_fake_edge(fake), fake) for fake in fakes] # Init the DisjointSet subgraphs = UnionFind() assert len(network.nodes()) > 1, \ "network must have more than 1 node" if single_network: # just union all nodes to a single parent nodes = network.nodes() # add parent parent = nodes[0] subgraphs.add_component(parent, budget=network.node[parent]['budget']) for node in nodes[1:]: subgraphs.add_component(node, budget=network.node[node]['budget']) # The existing grid nodes are on the grid (so distance is 0) subgraphs.union(parent, node, 0) else: # Build the subnet components # Get the network components to init budget centers subnets = nx.connected_components(network) for sub in subnets: # union all nodes to parent of subnet parent = sub[0] subgraphs.add_component(parent, budget=network.node[parent]['budget']) # Merge remaining nodes with component for node in sub[1:]: subgraphs.add_component(node, budget=network.node[node]['budget']) # The existing grid nodes are on the grid (so distance is 0) subgraphs.union(parent, node, 0) # setup merged graph to be populated with fake nodes merged = GeoGraph(demand_nodes.srs, demand_nodes.coords, data=demand_nodes) # merge fakes in for ((u, v), fake) in edge_fakes: # Make sure something wonky isn't going on assert(subgraphs[u] == subgraphs[v]) # Add the fake node to the big net merged.add_node(fake, budget=np.inf) merged.coords[fake] = grid_with_fakes.coords[fake] # Merge the fake node with the grid subgraph subgraphs.add_component(fake, budget=np.inf) subgraphs.union(fake, u, 0) return merged, subgraphs, rtree
def build_network(demand_nodes, existing=None, min_node_count=2, single_network=True, network_algorithm='mod_boruvka', one_based=False ): """ project demand nodes onto optional existing supply network and return the 'optimized' network Args: demand_nodes: GeoGraph of demand nodes existing: GeoGraph of existing grid (assumes node ids don't conflict with demand_nodes min_node_count: minimum number of nodes allowed in a subgraph of the result network_algorithm: Algorithm from ALGOS to run one_based: Whether result GeoGraph's nodes should be one_based (if not, they are 0 based) Returns: msf: GeoGraph of minimum spanning forest proposed by the chosen network algorithm existing: The existing grid GeoGraph (None if it doesn't exist) """ geo_graph = subgraphs = rtree = None if existing: log.info("merging network and nodes") geo_graph, subgraphs, rtree = \ merge_network_and_nodes(existing, demand_nodes, single_network=single_network) else: geo_graph = demand_nodes log.info("running {} on {} demand nodes and {} total nodes".format( network_algorithm, len(demand_nodes), len(geo_graph))) # now run the selected algorithm network_algo = NetworkerRunner.ALGOS[network_algorithm] result_geo_graph = network_algo(geo_graph, subgraphs=subgraphs, rtree=rtree) # TODO: Remove unreferenced fake nodes? # now filter out subnetworks via minimum node count # TODO: update union_all to support GeoGraph? filtered_graph = nx.union_all(filter( lambda sub: len(sub.node) >= min_node_count, nx.connected_component_subgraphs(result_geo_graph))) # map coords back to geograph # NOTE: explicit relabel to int as somewhere in filtering above, some # node ids are set to numpy types which screws up comparisons to tuples # in write op # NOTE: relabeling nodes in-place here drops node attributes for some # reason so create a copy for now def id_label(i): id = int(i+1) if one_based else int(i) return id msf = None if filtered_graph: coords = {id_label(i): result_geo_graph.coords[i] for i in filtered_graph} relabeled = nx.relabel_nodes(filtered_graph, {i: id_label(i) for i in filtered_graph}, copy=True) msf = GeoGraph(result_geo_graph.srs, coords=coords, data=relabeled) log.info("filtered result has {} nodes and {} edges".format( len(msf.nodes()), len(msf.edges()))) return msf
def test_project_xyz_vs_geo(): """ ensure that project_onto works the same with xyz vs lat_lon points """ net_coords = {'grid-1': [0.0, 0.0], 'grid-2': [10.0, 10.0]} net_edges = [('grid-1', 'grid-2')] node_coords = {0: [5.0, 5.0]} g_net = GeoGraph(gm.PROJ4_LATLONG, net_coords, data=net_edges) g_nodes = GeoGraph(gm.PROJ4_LATLONG, node_coords) g_project = g_net.project_onto(g_nodes, spherical_accuracy=True) g_net_xyz = GeoGraph(gm.PROJ4_LATLONG, g_net.lon_lat_to_cartesian_coords(), data=net_edges) g_nodes_xyz = GeoGraph(gm.PROJ4_LATLONG, g_nodes.lon_lat_to_cartesian_coords()) g_project_xyz = g_net_xyz.project_onto(g_nodes_xyz, spherical_accuracy=True) g_project_coords_ll = g_project_xyz.cartesian_to_lon_lat() def round_coords(coords, round_precision=8): return {i: tuple(map(lambda c: round(c, round_precision), coord)) for i, coord in coords.items()} assert(round_coords(g_project_coords_ll) == round_coords(g_project.coords))
def merge_network_and_nodes(network, demand_nodes, single_network=True, spherical_accuracy=False): """ merge the network and nodes GeoGraphs to set up the Graph, UnionFind (DisjoinSet), and RTree datastructures for use in network algorithms Args: network: graph representing existing network (assumes node ids don't conflict with net (demand) nodes) demand_nodes: graph of nodes representing demand single_network: whether subgraphs of network are unioned into a single network spherical_accuracy: Whether to connect nodes to network on a sphere Returns: graph: graph with demand nodes and their nearest nodes to the existing network (i.e. 'fake' nodes) subgraphs: UnionFind datastructure populated with fake nodes and associated with the appropriate connected component or the entire subgraph (depending on ``single_subgraph`` param) rtree: spatial index populated with the edges from the existing network """ # project demand nodes onto network rtree = network.get_rtree_index() grid_with_fakes = network.project_onto(demand_nodes, rtree_index=rtree, spherical_accuracy=spherical_accuracy) # get only the fake nodes and the associated network edges demand_node_set = set(demand_nodes.nodes()) net_plus_demand = set(network.nodes()).union(demand_node_set) fakes = set(grid_with_fakes.nodes()) - net_plus_demand def get_fake_edge(node): """ fake node should only have 2 neighbors from the existing network that is the nearest edge """ return tuple(set(grid_with_fakes.neighbors(node)) - demand_node_set) edge_fakes = [(get_fake_edge(fake), fake) for fake in fakes] # Init the DisjointSet subgraphs = UnionFind() assert len(network.nodes()) > 1, \ "network must have more than 1 node" if single_network: # just union all nodes to a single parent nodes = network.nodes() # add parent parent = nodes[0] subgraphs.add_component(parent, budget=network.node[parent]['budget']) for node in nodes[1:]: subgraphs.add_component(node, budget=network.node[node]['budget']) # The existing grid nodes are on the grid (so distance is 0) subgraphs.union(parent, node, 0) else: # Build the subnet components # Get the network components to init budget centers subnets = nx.connected_components(network) for sub in subnets: # union all nodes to parent of subnet sub_list = list(sub) parent = sub_list[0] subgraphs.add_component(parent, budget=network.node[parent]['budget']) # Merge remaining nodes with component for node in sub_list[1:]: subgraphs.add_component(node, budget=network.node[node]['budget']) # The existing grid nodes are on the grid (so distance is 0) subgraphs.union(parent, node, 0) # setup merged graph to be populated with fake nodes merged = GeoGraph(demand_nodes.srs, demand_nodes.coords, data=demand_nodes) # merge fakes in for ((u, v), fake) in edge_fakes: # Make sure something wonky isn't going on assert(subgraphs[u] == subgraphs[v]) # Add the fake node to the big net # NOTE: fake nodes always have np.inf budget merged.add_node(fake, budget=np.inf) merged.coords[fake] = grid_with_fakes.coords[fake] # Merge the fake node with the grid subgraph subgraphs.add_component(fake, budget=np.inf) subgraphs.union(fake, u, 0) return merged, subgraphs, rtree
def build_network(demand_nodes, existing=None, min_node_count=2, single_network=True, network_algorithm='mod_boruvka', spherical_accuracy=False, one_based=False): """ project demand nodes onto optional existing supply network and return the 'optimized' network Args: demand_nodes: GeoGraph of demand nodes existing: GeoGraph of existing grid (assumes node ids don't conflict with demand_nodes min_node_count: minimum number of nodes allowed in a subgraph of the result network_algorithm: Algorithm from ALGOS to run spherical_accuracy: Whether to connect nodes to network on a sphere one_based: Whether result GeoGraph's nodes should be one_based (if not, they are 0 based) Returns: msf: GeoGraph of minimum spanning forest proposed by the chosen network algorithm existing: The existing grid GeoGraph (None if it doesn't exist) """ geo_graph = subgraphs = rtree = None if existing: log.info("merging network and nodes") geo_graph, subgraphs, rtree = \ merge_network_and_nodes(existing, demand_nodes, single_network=single_network, spherical_accuracy=spherical_accuracy) else: geo_graph = demand_nodes log.info("running {} on {} demand nodes and {} total nodes".format( network_algorithm, len(demand_nodes), len(geo_graph))) # now run the selected algorithm network_algo = NetworkerRunner.ALGOS[network_algorithm] result_geo_graph = network_algo(geo_graph, subgraphs=subgraphs, rtree=rtree) filtered_graph = filter_min_node_subnetworks(result_geo_graph, min_node_count) # map coords back to geograph # NOTE: explicit relabel to int as somewhere in filtering above, some # node ids are set to numpy types which screws up comparisons to tuples # in write op # NOTE: relabeling nodes in-place here drops node attributes for some # reason so create a copy for now def id_label(i): id = int(i+1) if one_based else int(i) return id msf = GeoGraph(result_geo_graph.srs) if filtered_graph: coords = {id_label(i): result_geo_graph.coords[i] for i in filtered_graph} relabeled = nx.relabel_nodes(filtered_graph, {i: id_label(i) for i in filtered_graph}, copy=True) msf = GeoGraph(result_geo_graph.srs, coords=coords, data=relabeled) log.info("filtered result has {} nodes and {} edges".format( len(msf.nodes()), len(msf.edges()))) return msf
def nodes_plus_grid(): """ Return: nodes as graph and grid as UnionFind/Rtree combo This example input demonstrates the "more" optimal nature of mod_boruvka vs mod_kruskal. 2(10) | 1(4) | sqrt(5){ /| | / | | / | }3 | } 5 (2)0 | | 1{| | | +-+-3-+-4-+-+-+-+-+-+-+5-+-+-+ <-- existing grid In this case, all nodes will be connected via either algorithm, but the graph produced by mod_kruskal will have edge (4,1) whereas mod_boruvka will produce a graph with edge (0,1). Therefore, the mod_boruvka graph is more optimal. """ mv_max_values = [2, 4, 10] coords = np.array([[0.0, 1.0], [1.0, 3.0], [10.0, 5.0]]) coords_dict = dict(enumerate(coords)) nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, coords=coords_dict) nx.set_node_attributes(nodes, 'budget', dict(enumerate(mv_max_values))) grid_coords = np.array([[-5.0, 0.0], [15.0, 0.0]]) grid = GeoGraph(gm.PROJ4_FLAT_EARTH, {'grid-' + str(n): c for n, c in enumerate(grid_coords)}) nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()}) grid.add_edges_from([('grid-0', 'grid-1')]) # now find projections onto grid rtree = grid.get_rtree_index() projected = grid.project_onto(nodes, rtree_index=rtree) projected.remove_nodes_from(grid) projected.remove_nodes_from(nodes) # populate disjoint set of subgraphs subgraphs = UnionFind() # only one connected component, so just associate all nodes # with first node of grid parent = grid.nodes()[0] subgraphs.add_component(parent, budget=grid.node[parent]['budget']) for node in grid.nodes()[1:]: subgraphs.add_component(node, budget=grid.node[node]['budget']) subgraphs.union(parent, node, 0) # and the projected "fake" nodes for node in projected.nodes(): subgraphs.add_component(node, budget=np.inf) subgraphs.union(parent, node, 0) # add projected nodes to node set nodes.add_nodes_from(projected, budget=np.inf) # merge coords nodes.coords = dict(nodes.coords, **projected.coords) return nodes, subgraphs, rtree
def nodes_plus_grid(): """ Return: nodes as graph and grid as UnionFind/Rtree combo This example input demonstrates the "more" optimal nature of mod_boruvka vs mod_kruskal. 2(10) | 1(4) | sqrt(5){ /| | / | | / | }3 | } 5 (2)0 | | 1{| | | +-+-3-+-4-+-+-+-+-+-+-+5-+-+-+ <-- existing grid In this case, all nodes will be connected via either algorithm, but the graph produced by mod_kruskal will have edge (4,1) whereas mod_boruvka will produce a graph with edge (0,1). Therefore, the mod_boruvka graph is more optimal. """ mv_max_values = [2, 4, 10] coords = np.array([[0.0, 1.0], [1.0, 3.0], [10.0, 5.0]]) coords_dict = dict(enumerate(coords)) nodes = GeoGraph(gm.PROJ4_FLAT_EARTH, coords=coords_dict) nx.set_node_attributes(nodes, 'budget', dict(enumerate(mv_max_values))) grid_coords = np.array([[-5.0, 0.0], [15.0, 0.0]]) grid = GeoGraph(gm.PROJ4_FLAT_EARTH, {'grid-' + str(n): c for n, c in enumerate(grid_coords)}) nx.set_node_attributes(grid, 'budget', {n: 0 for n in grid.nodes()}) grid.add_edges_from([('grid-0', 'grid-1')]) # now find projections onto grid rtree = grid.get_rtree_index() projected = grid.project_onto(nodes, rtree_index=rtree) projected.remove_nodes_from(grid) projected.remove_nodes_from(nodes) # populate disjoint set of subgraphs subgraphs = UnionFind() # only one connected component, so just associate all nodes # with first node of grid parent = grid.nodes()[0] subgraphs.add_component(parent, budget=grid.node[parent]['budget']) for node in grid.nodes()[1:]: subgraphs.add_component(node, budget=grid.node[node]['budget']) subgraphs.union(parent, node, 0) # and the projected "fake" nodes for node in projected.nodes(): subgraphs.add_component(node, budget=np.inf) subgraphs.union(parent, node, 0) # add projected nodes to node set nodes.add_nodes_from(projected, budget=np.inf) # merge coords nodes.coords = dict(nodes.coords, **projected.coords) return nodes, subgraphs, rtree