def route_cnx(file_name, source_id=None, target_id=None, out_file=None, visualize=False, benchmark=None): from limic.util import start, end, load_pickled, save_path, save_pickled start("Loading from", file_name) g = load_pickled(file_name) end() start("Routing using condensed NX") if benchmark and out_file: routes = [] for i in range(int(benchmark)): path = astar_cnx(g, None, None, routes) elif benchmark: for source, target in load_pickled(benchmark): path = astar_cnx(g, (source, ), (target, )) else: path = astar_cnx(g, source_id, target_id) end() if benchmark and out_file: start("Saving routes to", out_file) save_pickled(out_file, routes) end() elif not benchmark: start("Saving path to", out_file) save_path(path, out_file, visualize) end()
def route_direct(file_name, source_id=None, target_id=None, out_file=None, overpass_url=None, disk_cache=False, visualize=False): from limic.util import start, end, status, file_size, load_pickled, save_pickled, save_path, options, replace if disk_cache: start("Using disk cache", file_name) set_option('disk_cache', file_name) from limic.overpass import region, set_server if disk_cache: status("OK") from os.path import exists if not disk_cache and exists(file_name): start("Loading", file_name) region.backend._cache = load_pickled(file_name) end('') file_size(file_name) len_cache = len(region.backend._cache) start("Routing using direct algorithm") set_server(overpass_url) path = astar_direct(source_id, target_id) end() start("Saving path to", out_file) save_path(path, out_file, visualize) end() if not disk_cache and len_cache != len(region.backend._cache): file_name_tmp = file_name + ".tmp" start("Saving to", file_name, "via", file_name_tmp) save_pickled(file_name_tmp, region.backend._cache) replace(file_name_tmp, file_name) end('') file_size(file_name)
def prune_nx(file_name_in, file_name_out, polygon, overpass_url): from limic.util import start, end, file_size, status, save_pickled, load_pickled, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading from", file_name_in) g = load_pickled(file_name_in) end('') file_size(file_name_in) polygon = list(map(float, polygon)) polygon = list(zip(polygon[::2], polygon[1::2])) if not overpass_url: from limic.util import kdtree, nodes_in_geometry start("Building kd-tree from nodes") tree = kdtree(g.nodes(), get_latlon=lambda x: (x[1], x[2])) end() start("Querying tree for nodes in polygon") nodes = nodes_in_geometry(tree, polygon) else: from limic.overpass import nodes_in_geometry, set_server start("Query server for nodes in polygon") set_server(overpass_url) nodes = nodes_in_geometry(polygon) end('') status(len(nodes)) start("Pruning graph") h = prune_ids_nx(g, nodes) end() start("Saving to", file_name_out) save_pickled(file_name_out, h) end('') file_size(file_name_out)
def extract_osm_pre(file_name_in, file_name_out, white="{'power':'line'}", black="{'power':'substation'}", conserve_mem=False): from limic.util import start, end, save_pickled, file_size, check_overwrite if not check_overwrite(file_name_in, file_name_out): return white, black = list(eval(white).items()), list(eval(black).items()) lim = osm_pre(file_name_in, white, black, conserve_mem) start("Saving data to", file_name_out) save_pickled(file_name_out, lim) end('') file_size(file_name_out)
def convert_npz_nx(file_name_in, file_name_out, penalize=20): from limic.util import start, end, file_size, save_pickled, load_npz, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading graph from", file_name_in) g = load_npz(file_name_in) end('') file_size(file_name_in) start("Transforming to NX format") h = transform_npz_nx(g, penalize) end() start("Saving to", file_name_out) save_pickled(file_name_out, h) end('') file_size(file_name_out)
def merge_cache(file_names, file_name_out): from limic.util import start, end, file_size, status, load_pickled, save_pickled, check_overwrites if not check_overwrites(file_names, file_name_out): return g = {} for file_name_in in file_names: start("Loading cache from", file_name_in) h = load_pickled(file_name_in) end('') file_size(file_name_in) start("Adding", len(h), "entries") g.update(h) end('') status(len(g)) start("Saving merged cache to", file_name_out) save_pickled(file_name_out, g) end('') file_size(file_name_out)
def shutdown(sig, frame): nonlocal interrupt interrupt += 1 print("Shutting down ...") for f in fs: f.cancel() print("Cancelled all futures ...") running = len(fs) total = running while running: print("Waiting for", running, "processes to shut down ...") wait(fs, timeout=60) running = sum(0 if f.done() else 1 for f in fs) if len_cache != len(region.backend._cache): file_name_tmp = file_name + "." + str(interrupt) start("Emergency saving to", file_name_tmp) save_pickled(file_name_tmp, region.backend._cache) end('') file_size(file_name_tmp)
def merge_nx(file_names, file_name_out): from limic.util import start, end, file_size, status, load_pickled, save_pickled, check_overwrites from networkx import Graph if not check_overwrites(file_names, file_name_out): return g = Graph() for file_name_in in file_names: start("Loading graph from", file_name_in) h = load_pickled(file_name_in) end('') file_size(file_name_in) start("Adding", h.number_of_edges(), "edges") for from_node, to_node, data in h.edges(data=True): g.add_edge(from_node, to_node, **data) end('') status(g.number_of_edges()) start("Saving merged graph to", file_name_out) save_pickled(file_name_out, g) end('') file_size(file_name_out)
def convert_dbm_cache(file_name_in, file_name_out): from limic.util import start, end, file_size, status, save_pickled, check_overwrite from dbm.gnu import open as dopen from pickle import loads if not check_overwrite(file_name_in, file_name_out): return start("Opening database", file_name_in) db = dopen(file_name_in, "r") end('') file_size(file_name_in) start("Converting to dictionary") d = {} for key in db.keys(): d[key.decode("utf-8")] = loads(db[key]) db.close() end() start("Saving to", file_name_out) save_pickled(file_name_out, d) end('') file_size(file_name_out)
def select_nx(file_name_in,file_name_out,polygon,overpass_url): from limic.util import start, end, file_size, status, save_pickled, load_pickled, check_overwrite from limic.overpass import nodes_in_geometry, set_server if not check_overwrite(file_name_in,file_name_out): return start("Loading from",file_name_in) g = load_pickled(file_name_in) end('') file_size(file_name_in) start("Query server for nodes in polygon") set_server(overpass_url) nodes = nodes_in_geometry(zip(polygon[::2], polygon[1::2])) end('') status(len(nodes)) start("Selecting area from graph") h = select_ids_nx(g,nodes) end() start("Saving to",file_name_out) save_pickled(file_name_out,h) end('') file_size(file_name_out)
def condense(file_name_in, file_name_out, lengths=False, paths=False): from limic.util import start, end, file_size, status, save_pickled, load_pickled, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading from", file_name_in) g = load_pickled(file_name_in) end('') file_size(file_name_in) start("Condensing edges") h = cnx(g) end() if lengths: start("Computing lengths") h.compute_lengths() end() if paths: start("Computing paths") h.compute_paths() end() start("Saving to", file_name_out) save_pickled(file_name_out, h) end('') file_size(file_name_out)
from limic.util import start, end, file_size, status, load_pickled, save_pickled from sys import argv if __name__ == "__main__": file_name_in = argv[1] file_name_out = argv[2] start("Loading graph from", file_name_in) g = load_pickled(file_name_in) end('') status(len(g.edges()), end=' ') file_size(file_name_in) start("Cleaning up graph") for u, v, d in g.edges(data=True): d['type'] = -1 if d['air'] else 0 del d['air'] end() start("Saving graph to", file_name_out) save_pickled(file_name_out, g) end('') file_size(file_name_out)
def fill(overpass_url, file_name=None, area=None, around=1000, eps=0.01, safe_dist=100, penalize=20, max_workers=None): from limic.overpass import set_server, pylon, region, get_towers_by_area from limic.util import start, end, file_size, status, load_pickled, save_pickled, options, replace, options from networkx import Graph, relabel_nodes from os import cpu_count from os.path import exists from concurrent.futures import ThreadPoolExecutor, wait from signal import signal, SIGINT if not area and not file_name: if options.parser: options.parser.error("specify at least one of --area or CACHE") else: status("ERROR: specify at least area or cache name!") from sys import exit exit(-1) if not area: area = file_name.split(".")[1] if not file_name: file_name = "cache." + area if not max_workers: max_workers = cpu_count() * 4 start("Number of workers") status(max_workers) if exists(file_name): start("Loading", file_name) region.backend._cache = load_pickled(file_name) end('') file_size(file_name) len_cache = len(region.backend._cache) start("Querying overpass for", area) set_server(overpass_url) towers = get_towers_by_area(area) end() fs = [] executor = ThreadPoolExecutor(max_workers=max_workers) interrupt = 0 def shutdown(sig, frame): nonlocal interrupt interrupt += 1 print("Shutting down ...") for f in fs: f.cancel() print("Cancelled all futures ...") running = len(fs) total = running while running: print("Waiting for", running, "processes to shut down ...") wait(fs, timeout=60) running = sum(0 if f.done() else 1 for f in fs) if len_cache != len(region.backend._cache): file_name_tmp = file_name + "." + str(interrupt) start("Emergency saving to", file_name_tmp) save_pickled(file_name_tmp, region.backend._cache) end('') file_size(file_name_tmp) signal(SIGINT, shutdown) options.failed = True while options.failed: options.failed = False for tower in towers: fs.append( executor.submit(cache_tower, tower, around, eps, safe_dist, penalize)) running = len(fs) total = running while running: print("Waiting for", running, "out of", total, "processes ...") wait(fs, timeout=60) running = sum(0 if f.done() else 1 for f in fs) if len_cache != len(region.backend._cache): file_name_tmp = file_name + ".tmp" start("Saving to", file_name, "via", file_name_tmp) save_pickled(file_name_tmp, region.backend._cache) replace(file_name_tmp, file_name) end('') file_size(file_name)
def osm_post(lim, file_name_out, around=1000, eps=0.01, safe_dist=100, penalize=20): from limic.util import start, end, status, file_size, load_pickled, distance, save_pickled from scipy.spatial import cKDTree as KDTree from networkx import Graph, astar_path_length from pyproj import CRS, Transformer from itertools import chain from limic.overpass import intersect, pylon lines, substations, towers, id2tower, id2node, id2lines, id2types = lim start("Building KD-tree from white nodes") from limic.util import kdtree towers_tree = kdtree(towers, get_latlon=lambda x: x.latlon) end('') status(len(towers)) start("Deleting black nodes") to_delete = set() from limic.util import nodes_in_geometry for substation in substations: to_delete.update( nodes_in_geometry(towers_tree, list(map(lambda x: id2node[x], substation)))) towers = [tower for tower in towers if tower not in to_delete] end('') status(len(towers)) start("Building initial graph") g = Graph() g.add_nodes_from(towers) for line in lines: line_nodes = list(map(lambda x: id2tower[x], line)) for from_node, to_node in zip(line_nodes, line_nodes[1:]): if from_node in to_delete or to_node in to_delete: continue w = distance(from_node.latlon, to_node.latlon) g.add_edge(from_node, to_node, weight=w, type=id2types[from_node.id]) end('') status(len(g.nodes()), end='/') status(len(g.edges())) start("Finding neighbours within " + str(around) + "m") towers_tree = kdtree(towers, get_latlon=lambda x: x.latlon) end('') neighbour_indices, neighbours = towers_tree.get_neighbours(around=1000) end() start("Computing non-logical intersections") tower2index = {} for i, t in zip(range(len(towers)), towers): tower2index[t] = i for k, v in id2lines.items(): id2lines[k] = tuple(map(tuple, v)) end('') segments = set() for u, v in g.edges(): this = (u, v) if u < v else (v, u) ui, vi = tower2index[u], tower2index[v] lines = set() lines.update(id2lines[u.id]) lines.update(id2lines[v.id]) for neighbour in chain(neighbours[ui], neighbours[vi]): if neighbour == u or neighbour == v: continue if not lines.intersection(id2lines[neighbour.id]): for nn in g.neighbors(neighbour): other = (neighbour, nn) if neighbour < nn else (nn, neighbour) segments.add(tuple(sorted((this, other)))) end('') status(len(segments), end=' ') neighbours2intersection = {} minusid = 0 latlon2id = {} segments2intersections = {} for (t1, t2), (t3, t4) in segments: res = intersect(t1.latlon, t2.latlon, t3.latlon, t4.latlon, eps=eps, no_tu=False) if res: intersection, (t, u) = res if not intersection in latlon2id: minusid -= 1 latlon2id[intersection] = minusid segments2intersections.setdefault((t1, t2), []).append( (t, latlon2id[intersection], intersection)) segments2intersections.setdefault((t3, t4), []).append( (u, latlon2id[intersection], intersection)) end('') status(-minusid, end=' ') for (u, v), intersections in segments2intersections.items(): intersections.sort() g.remove_edge(u, v) type = id2types[u.id] assert (type == id2types[v.id]) seq = [u] for _, id, latlon in intersections: seq.append(pylon(id, latlon)) seq.append(v) for from_node, to_node in zip(seq, seq[1:]): w = distance(from_node.latlon, to_node.latlon) g.add_edge(from_node, to_node, weight=w, type=type) end() start("Adding routing through air") airs = set() for ns in neighbours: n = ns[0] for m in ns[1:]: if not g.has_edge(n, m): airs.add((n, m)) end('') for n, m in airs: w = penalize * distance(n.latlon, m.latlon) g.add_edge(n, m, weight=w, type=-1) end('') status(len(g.nodes()), end='/') status(len(g.edges())) from networkx import relabel_nodes start("Prune redundant edges (incomplete)") prune_incomplete(g) end('') status(len(g.edges())) start("Prune redundant edges (complete)") prune_complete(g) end('') status(len(g.edges())) start("Cleaning up graph") relabel = dict( map( lambda tower: (tower, (tower.id, tower.latlon[0], tower.latlon[1])), g.nodes())) relabel_nodes(g, relabel, copy=False) end() start("Saving graph to", file_name_out) save_pickled(file_name_out, g) end('') file_size(file_name_out)
def extract_cache(file_name_in, file_name_out, overpass_url, area=None, around=1000, eps=0.01, safe_dist=100, penalize=20): #from limic.overpass import distance, find_all_neighbours, is_safe, set_server, pylon, region, get_towers_by_area #from limic.util import start, end, file_size, status, load_pickled, save_pickled, replace, check_overwrite from networkx import Graph if not check_overwrite(file_name_in, file_name_out): return start("Loading", file_name_in) region.backend._cache = load_pickled(file_name_in) len_cache = len(region.backend._cache) end('') file_size(file_name_in) if not area: area = file_name_in.split(".")[1] start("Querying overpass for", area) set_server(overpass_url) towers = get_towers_by_area(area) end() start("Building safe nodes") g = Graph() for tower in towers: if is_safe(tower, safe_dist): g.add_node(tower) # else: # if verbosity >= 2: print("NOT safe!") end('') total = len(g.nodes()) status(total) start("Building edges") build_edges(g, find_all_neighbours, around, eps, safe_dist, penalize) end('') status(len(g.edges())) if len_cache != len(region.backend._cache): file_name_tmp = file_name_in + ".tmp" start("Saving to", file_name_in, "via", file_name_tmp) save_pickled(file_name_tmp, region.backend._cache) replace(file_name_tmp, file_name_in) end('') file_size(file_name_in) from limic.util import start, end, status, file_size, save_pickled from networkx import relabel_nodes start("Prune redundant edges (incomplete)") prune_incomplete(g) end('') status(len(g.edges())) start("Prune redundant edges (complete)") prune_complete(g) end('') status(len(g.edges())) start("Cleaning up graph") relabel = dict( map( lambda tower: (tower, (tower.id, tower.latlon[0], tower.latlon[1])), g.nodes())) relabel_nodes(g, relabel, copy=False) for u, v, d in g.edges(data=True): d['type'] = -1 if d['type'] else 0 end() start("Saving graph to", file_name_out) save_pickled(file_name_out, g) end('') file_size(file_name_out)