def _get_edge_groups_to_simplify(G, no_processes=1): # first identify all the nodes that are endpoints endpoints = set( parallel.multiprocess_wrap( data={node: {'successors': set(G.successors(node)), 'predecessors': set(G.predecessors(node))} for node in G.nodes}, split=parallel.split_dict, apply=_is_endpoint, combine=parallel.combine_list, processes=no_processes) ) logging.info(f"Identified {len(endpoints)} edge endpoints") path_start_points = list(G.out_edges(endpoints)) logging.info(f"Identified {len(path_start_points)} possible paths") return parallel.multiprocess_wrap( data=path_start_points, split=parallel.split_list, apply=_build_paths, combine=parallel.combine_list, processes=no_processes, endpoints=endpoints, neighbours={node: set(G.neighbors(node)) for node in G.nodes} )
def create_s2_indexed_osm_graph(response_jsons, config, num_processes, bidirectional): logging.info('Creating networkx graph from OSM data') elements = [] for response_json in response_jsons: elements.extend(response_json['elements']) logging.info('OSM: Extract Nodes and Paths from OSM data') nodes = {} paths = {} for osm_data in response_jsons: nodes_temp, paths_temp = osmnx_customised.parse_osm_nodes_paths(osm_data, config) for key, value in nodes_temp.items(): nodes[key] = value for key, value in paths_temp.items(): paths[key] = value logging.info('OSM: Add each OSM way (aka, path) to the OSM graph') edges = parallel.multiprocess_wrap( data=paths, split=parallel.split_dict, apply=osmnx_customised.return_edges, combine=parallel.combine_list, processes=num_processes, config=config, bidirectional=bidirectional) return nodes, edges
def test_multiprocess_wrap_function_processing_dict_data(): input = dict(zip(range(200), ['a'] * 200)) output = parallel.multiprocess_wrap(data=input, split=parallel.split_dict, apply=dict_to_list_function, combine=parallel.combine_list, processes=1) assert output == list(range(200))
def test_multiprocess_wrapping_with_custom_simple_split_method(): # i.e. doesnt rely on number of processes input = dict(zip(range(10), ['a'] * 10)) output = parallel.multiprocess_wrap(data=input, split=custom_simple_split_method, apply=dict_to_list_function, combine=parallel.combine_list, processes=2) assert output == list(range(10))
def read_osm(osm_file_path, osm_read_config, num_processes: int = 1, epsg=None): """ Reads OSM data into a graph of the Network object :param osm_file_path: path to .osm or .osm.pbf file :param osm_read_config: config file (see configs folder in genet for examples) which informs for example which highway types to read (in case of road network) and what modes to assign to them :param num_processes: number of processes to split parallelisable operations across :param epsg: projection for the output Network, e.g. 'epsg:27700'. If not provided, defaults to epsg:4326 :return: genet.Network object """ if epsg is None: epsg = 'epsg:4326' config = osm_reader.Config(osm_read_config) n = core.Network(epsg) nodes, edges = osm_reader.generate_osm_graph_edges_from_file( osm_file_path, config, num_processes) nodes_and_attributes = parallel.multiprocess_wrap( data=nodes, split=parallel.split_dict, apply=osm_reader.generate_graph_nodes, combine=parallel.combine_dict, epsg=epsg, processes=num_processes ) reindexing_dict, nodes_and_attributes = n.add_nodes(nodes_and_attributes, ignore_change_log=True) edges_attributes = parallel.multiprocess_wrap( data=edges, split=parallel.split_list, apply=osm_reader.generate_graph_edges, combine=parallel.combine_list, reindexing_dict=reindexing_dict, nodes_and_attributes=nodes_and_attributes, config_path=osm_read_config, processes=num_processes ) n.add_edges(edges_attributes, ignore_change_log=True) logging.info('Deleting isolated nodes which have no edges.') n.remove_nodes(list(nx.isolates(n.graph))) return n
def reproject(self, new_epsg, processes=1): """ Changes projection of the element to new_epsg :param new_epsg: 'epsg:1234' :return: """ if self.epsg != new_epsg: g = self.graph() reprojected_node_attribs = parallel.multiprocess_wrap( data=dict(g.nodes(data=True)), split=parallel.split_dict, apply=mod_schedule.reproj_stops, combine=parallel.combine_dict, processes=processes, new_epsg=new_epsg) nx.set_node_attributes(self._graph, reprojected_node_attribs) self.epsg = new_epsg
def simplify_graph(n, no_processes=1): """ MONKEY PATCH OF OSMNX'S GRAPH SIMPLIFICATION ALGO Simplify a graph's topology by removing interstitial nodes. Simplify graph topology by removing all nodes that are not intersections or dead-ends. Create an edge directly between the end points that encapsulate them, but retain the geometry of the original edges, saved as attribute in new edge. Parameters ---------- n: genet.Network object no_processes: number of processes to split some of the processess across Returns ------- None, updates n.graph, indexing and schedule routes. Adds a new attribute to n that records map between old and new link indices """ logging.info("Begin simplifying the graph") initial_node_count = len(list(n.graph.nodes())) initial_edge_count = len(list(n.graph.edges())) logging.info('Generating paths to be simplified') # generate each path that needs to be simplified edges_to_simplify = _get_edge_groups_to_simplify(n.graph, no_processes=no_processes) logging.info(f'Found {len(edges_to_simplify)} paths to simplify.') indexed_paths_to_simplify = dict(zip(n.generate_indices_for_n_edges(len(edges_to_simplify)), edges_to_simplify)) indexed_paths_to_simplify = _assemble_path_data(n, indexed_paths_to_simplify) nodes_to_remove = set() for k, data in indexed_paths_to_simplify.items(): nodes_to_remove |= set(data['nodes_to_remove']) n.remove_nodes(nodes_to_remove, ignore_change_log=True, silent=True) logging.info('Processing links for all paths to be simplified') links_to_add = parallel.multiprocess_wrap( data=indexed_paths_to_simplify, split=parallel.split_dict, apply=_process_path, combine=parallel.combine_dict, processes=no_processes ) logging.info('Adding new simplified links') # add links reindexing_dict = n.add_links(links_to_add, ignore_change_log=True)[0] # generate link simplification map between old indices and new, add changelog event for old_id, new_id in reindexing_dict.items(): indexed_paths_to_simplify[new_id] = indexed_paths_to_simplify[old_id] del indexed_paths_to_simplify[old_id] new_ids = list(indexed_paths_to_simplify.keys()) old_ids = [set(indexed_paths_to_simplify[_id]['ids']) for _id in new_ids] n.change_log.simplify_bunch(old_ids, new_ids, indexed_paths_to_simplify, links_to_add) del links_to_add # generate map between old and new ids n.link_simplification_map = {} for old_id_list, new_id in zip(old_ids, new_ids): for _id in old_id_list: n.link_simplification_map[_id] = new_id logging.info( f"Simplified graph: {initial_node_count} to {len(n.graph)} nodes, {initial_edge_count} to " f"{len(n.graph.edges())} edges") if n.schedule: logging.info("Updating the Schedule") # update stop's link reference ids new_stops_attribs = {} for node, link_ref_id in n.schedule._graph.nodes(data='linkRefId'): try: new_stops_attribs[node] = {'linkRefId': n.link_simplification_map[link_ref_id]} except KeyError: # Not all linkref ids would have changed pass nx.set_node_attributes(n.schedule._graph, new_stops_attribs) logging.info("Updated Stop Link Reference Ids") # update schedule routes for service_id, route in n.schedule.routes(): new_route = [] for link in route.route: updated_route_link = link if link in n.link_simplification_map: updated_route_link = n.link_simplification_map[link] if not new_route: new_route = [updated_route_link] elif new_route[-1] != updated_route_link: new_route.append(updated_route_link) route.route = new_route logging.info("Updated Network Routes")