def convert_nx_npz(file_name_in, file_name_out, indirect=False, rescale=False, penalize=20): from limic.util import start, end, file_size, load_pickled, save_npz, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading graph from", file_name_in) g = load_pickled(file_name_in) end('') file_size(file_name_in) if indirect: start("Transforming to GT format") i = transform_nx_gt(g, rescale) end() start("Transforming to NPZ format") h = transform_gt_npz(i, penalize) end() else: start("Transforming to NPZ format") h = transform_nx_npz(g, rescale) end() start("Saving to", file_name_out) save_npz(file_name_out, h) end('') file_size(file_name_out)
def render_npz(file_name_in, file_name_out, markers=False, lines=False, host="localhost", port=5000, prefix=""): from limic.util import start, end, load_npz, check_overwrite from numpy import column_stack if not check_overwrite(file_name_in, file_name_out): return start("Loading NPZ graph", file_name_in) g = load_npz(file_name_in) nodes = list(map(tuple, column_stack((g['ids'], g['lat'], g['long'])))) edges = [] if lines: id2edges = g['id2edges'] edges_weight = g['edges_weight'] edges_neighbor = g['edges_neighbor'] for index in range(len(nodes)): left = id2edges[index] right = id2edges[index + 1] me = nodes[index] for i in range(left, right): edges.append((me, nodes[edges_neighbor[i]], edges_weight[i])) end() return render(g, nodes, edges, file_name_out, markers, lines, host, port, prefix)
def prune_nx(file_name_in, file_name_out, polygon, overpass_url): from limic.util import start, end, file_size, status, save_pickled, load_pickled, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading from", file_name_in) g = load_pickled(file_name_in) end('') file_size(file_name_in) polygon = list(map(float, polygon)) polygon = list(zip(polygon[::2], polygon[1::2])) if not overpass_url: from limic.util import kdtree, nodes_in_geometry start("Building kd-tree from nodes") tree = kdtree(g.nodes(), get_latlon=lambda x: (x[1], x[2])) end() start("Querying tree for nodes in polygon") nodes = nodes_in_geometry(tree, polygon) else: from limic.overpass import nodes_in_geometry, set_server start("Query server for nodes in polygon") set_server(overpass_url) nodes = nodes_in_geometry(polygon) end('') status(len(nodes)) start("Pruning graph") h = prune_ids_nx(g, nodes) end() start("Saving to", file_name_out) save_pickled(file_name_out, h) end('') file_size(file_name_out)
def convert_cache_dbm(file_name_in, file_name_out): from limic.util import start, end, file_size, status, load_pickled, check_overwrite from dbm.gnu import open as dopen from os.path import exists from pickle import dumps if not check_overwrite(file_name_in, file_name_out): return start("Loading from", file_name_in) d = load_pickled(file_name_in) end('') file_size(file_name_in) start("Opening database", file_name_out) if not exists(file_name_out): db = dopen(file_name_out, "c") db.close() db = dopen(file_name_out, "c") end('') file_size(file_name_out) start("Computing set of entries to save") for key in db.keys(): del d[key.decode("utf-8")] status(len(d)) start("Saving entries to", file_name_out) for key, val in d.items(): db[key.encode("utf-8")] = dumps(val) db.close() end('') file_size(file_name_out)
def extract_osm_pre(file_name_in, file_name_out, white="{'power':'line'}", black="{'power':'substation'}", conserve_mem=False): from limic.util import start, end, save_pickled, file_size, check_overwrite if not check_overwrite(file_name_in, file_name_out): return white, black = list(eval(white).items()), list(eval(black).items()) lim = osm_pre(file_name_in, white, black, conserve_mem) start("Saving data to", file_name_out) save_pickled(file_name_out, lim) end('') file_size(file_name_out)
def convert_npz_nx(file_name_in, file_name_out, penalize=20): from limic.util import start, end, file_size, save_pickled, load_npz, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading graph from", file_name_in) g = load_npz(file_name_in) end('') file_size(file_name_in) start("Transforming to NX format") h = transform_npz_nx(g, penalize) end() start("Saving to", file_name_out) save_pickled(file_name_out, h) end('') file_size(file_name_out)
def convert_gt_npz(file_name_in, file_name_out, penalize=20): from limic.util import start, end, file_size, load_gt, save_npz, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading graph from", file_name_in) g = load_gt(file_name_in) end('') file_size(file_name_in) start("Initialzing id mapping and neighbours map") h = transform_gt_npz(g, penalize) end() start("Saving to", file_name_out) save_npz(file_name_out, h) end('') file_size(file_name_out)
def convert_nx_gt(file_name_in, file_name_out, rescale=False): from limic.util import start, end, file_size, load_pickled, save_gt, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading graph from", file_name_in) g = load_pickled(file_name_in) end('') file_size(file_name_in) start("Initialzing id mapping and neighbours map") h = transform_nx_gt(g, rescale) end() start("Saving to", file_name_out) save_gt(file_name_out, h) end('') file_size(file_name_out)
def render_nx(file_name_in, file_name_out, markers=False, lines=False, host="localhost", port=5000, prefix=""): from limic.util import start, end, load_pickled, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading NX graph", file_name_in) g = load_pickled(file_name_in) nodes = list(g.nodes()) edges = list(g.edges.data('weight')) end() return render(g, nodes, edges, file_name_out, markers, lines, host, port, prefix)
def extract_osm_post(file_name_in, file_name_out, around=1000, eps=0.01, safe_dist=100, penalize=20): from limic.util import start, end, file_size, load_pickled, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading filtered OSM data from", file_name_in) lim = load_pickled(file_name_in) end('') file_size(file_name_in) osm_post(lim, file_name_out, around=1000, eps=0.01, safe_dist=100, penalize=20)
def render_cnx(file_name_in, file_name_out, components=[], markers=False, lines=False, host="localhost", port=5000, prefix=""): from limic.util import start, end, load_pickled, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading CNX graph", file_name_in) cs, _, _ = load_pickled(file_name_in) end() for i in components if components else range(len(cs)): nodes = list(cs[i].nodes()) edges = list(cs[i].edges.data('weight')) file_out = file_name_out.split(".html")[0] + "." + str(i) + ".html" render(cs[i], nodes, edges, file_out, markers, lines, host, port, prefix)
def convert_dbm_cache(file_name_in, file_name_out): from limic.util import start, end, file_size, status, save_pickled, check_overwrite from dbm.gnu import open as dopen from pickle import loads if not check_overwrite(file_name_in, file_name_out): return start("Opening database", file_name_in) db = dopen(file_name_in, "r") end('') file_size(file_name_in) start("Converting to dictionary") d = {} for key in db.keys(): d[key.decode("utf-8")] = loads(db[key]) db.close() end() start("Saving to", file_name_out) save_pickled(file_name_out, d) end('') file_size(file_name_out)
def prune_gt(file_name_in, file_name_out, polygon, overpass_url): from limic.util import start, end, file_size, status, save_gt, load_gt, check_overwrite from limic.overpass import nodes_in_geometry, set_server if not check_overwrite(file_name_in, file_name_out): return start("Loading from", file_name_in) g = load_gt(file_name_in) end('') file_size(file_name_in) start("Query server for nodes in polygon") set_server(overpass_url) nodes = nodes_in_geometry(zip(polygon[::2], polygon[1::2])) end('') status(len(nodes)) start("Pruning graph") h = prune_ids_gt(g, nodes) end() start("Saving to", file_name_out) save_gt(file_name_out, h) end('') file_size(file_name_out)
def extract_osm(file_name_in, file_name_out, around=1000, eps=0.01, safe_dist=100, penalize=20, white="{'power':'line'}", black="{'power':'substation'}", conserve_mem=False): from setproctitle import setproctitle setproctitle(file_name_in) from limic.util import check_overwrite if not check_overwrite(file_name_in, file_name_out): return white, black = list(eval(white).items()), list(eval(black).items()) lim = osm_pre(file_name_in, white, black, conserve_mem) osm_post(lim, file_name_out, around=1000, eps=0.01, safe_dist=100, penalize=20)
def condense(file_name_in, file_name_out, lengths=False, paths=False): from limic.util import start, end, file_size, status, save_pickled, load_pickled, check_overwrite if not check_overwrite(file_name_in, file_name_out): return start("Loading from", file_name_in) g = load_pickled(file_name_in) end('') file_size(file_name_in) start("Condensing edges") h = cnx(g) end() if lengths: start("Computing lengths") h.compute_lengths() end() if paths: start("Computing paths") h.compute_paths() end() start("Saving to", file_name_out) save_pickled(file_name_out, h) end('') file_size(file_name_out)
def extract_cache(file_name_in, file_name_out, overpass_url, area=None, around=1000, eps=0.01, safe_dist=100, penalize=20): #from limic.overpass import distance, find_all_neighbours, is_safe, set_server, pylon, region, get_towers_by_area #from limic.util import start, end, file_size, status, load_pickled, save_pickled, replace, check_overwrite from networkx import Graph if not check_overwrite(file_name_in, file_name_out): return start("Loading", file_name_in) region.backend._cache = load_pickled(file_name_in) len_cache = len(region.backend._cache) end('') file_size(file_name_in) if not area: area = file_name_in.split(".")[1] start("Querying overpass for", area) set_server(overpass_url) towers = get_towers_by_area(area) end() start("Building safe nodes") g = Graph() for tower in towers: if is_safe(tower, safe_dist): g.add_node(tower) # else: # if verbosity >= 2: print("NOT safe!") end('') total = len(g.nodes()) status(total) start("Building edges") build_edges(g, find_all_neighbours, around, eps, safe_dist, penalize) end('') status(len(g.edges())) if len_cache != len(region.backend._cache): file_name_tmp = file_name_in + ".tmp" start("Saving to", file_name_in, "via", file_name_tmp) save_pickled(file_name_tmp, region.backend._cache) replace(file_name_tmp, file_name_in) end('') file_size(file_name_in) from limic.util import start, end, status, file_size, save_pickled from networkx import relabel_nodes start("Prune redundant edges (incomplete)") prune_incomplete(g) end('') status(len(g.edges())) start("Prune redundant edges (complete)") prune_complete(g) end('') status(len(g.edges())) start("Cleaning up graph") relabel = dict( map( lambda tower: (tower, (tower.id, tower.latlon[0], tower.latlon[1])), g.nodes())) relabel_nodes(g, relabel, copy=False) for u, v, d in g.edges(data=True): d['type'] = -1 if d['type'] else 0 end() start("Saving graph to", file_name_out) save_pickled(file_name_out, g) end('') file_size(file_name_out)