def convert_nx_npz(file_name_in,
file_name_out,
indirect=False,
rescale=False,
penalize=20):
from limic.util import start, end, file_size, load_pickled, save_npz, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading graph from", file_name_in)
g = load_pickled(file_name_in)
end('')
file_size(file_name_in)
if indirect:
start("Transforming to GT format")
i = transform_nx_gt(g, rescale)
end()
start("Transforming to NPZ format")
h = transform_gt_npz(i, penalize)
end()
else:
start("Transforming to NPZ format")
h = transform_nx_npz(g, rescale)
end()
start("Saving to", file_name_out)
save_npz(file_name_out, h)
end('')
file_size(file_name_out)
def route_direct(file_name,
source_id=None,
target_id=None,
out_file=None,
overpass_url=None,
disk_cache=False,
visualize=False):
from limic.util import start, end, status, file_size, load_pickled, save_pickled, save_path, options, replace
if disk_cache:
start("Using disk cache", file_name)
set_option('disk_cache', file_name)
from limic.overpass import region, set_server
if disk_cache:
status("OK")
from os.path import exists
if not disk_cache and exists(file_name):
start("Loading", file_name)
region.backend._cache = load_pickled(file_name)
end('')
file_size(file_name)
len_cache = len(region.backend._cache)
start("Routing using direct algorithm")
set_server(overpass_url)
path = astar_direct(source_id, target_id)
end()
start("Saving path to", out_file)
save_path(path, out_file, visualize)
end()
if not disk_cache and len_cache != len(region.backend._cache):
file_name_tmp = file_name + ".tmp"
start("Saving to", file_name, "via", file_name_tmp)
save_pickled(file_name_tmp, region.backend._cache)
replace(file_name_tmp, file_name)
end('')
file_size(file_name)
def download_osm(countries,url=None,show=False,max_workers=None):
from limic.util import start,end,file_size,status
from concurrent.futures import ProcessPoolExecutor, wait
countries, url = common(countries,url,show,osm=True)
if max_workers:
executor = ProcessPoolExecutor(max_workers=max_workers)
fs = []
start("Downloading OSM map data for"," ".join(countries))
for country in countries:
file_url = OSM_URL+country.replace("_","/")+"-latest.osm.bz2"
file_name = country+"-latest.osm.bz2"
if max_workers:
fs.append(executor.submit(download_file,file_url,file_name))
continue
start("Downloading OSM map data for",country)
file_name = download_file(file_url,file_name)
end('')
file_size(file_name)
if max_workers:
running = len(fs)
total = running
while running:
print("Waiting for",running,"out of",total,"processes ...")
wait(fs,timeout=10)
running = sum(0 if f.done() else 1 for f in fs)
for f in fs:
file_name = f.result()
status(file_name,end=':')
file_size(file_name,end=' ')
end()
def prune_nx(file_name_in, file_name_out, polygon, overpass_url):
from limic.util import start, end, file_size, status, save_pickled, load_pickled, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading from", file_name_in)
g = load_pickled(file_name_in)
end('')
file_size(file_name_in)
polygon = list(map(float, polygon))
polygon = list(zip(polygon[::2], polygon[1::2]))
if not overpass_url:
from limic.util import kdtree, nodes_in_geometry
start("Building kd-tree from nodes")
tree = kdtree(g.nodes(), get_latlon=lambda x: (x[1], x[2]))
end()
start("Querying tree for nodes in polygon")
nodes = nodes_in_geometry(tree, polygon)
else:
from limic.overpass import nodes_in_geometry, set_server
start("Query server for nodes in polygon")
set_server(overpass_url)
nodes = nodes_in_geometry(polygon)
end('')
status(len(nodes))
start("Pruning graph")
h = prune_ids_nx(g, nodes)
end()
start("Saving to", file_name_out)
save_pickled(file_name_out, h)
end('')
file_size(file_name_out)
def download_cache(countries,url=None,show=False):
from limic.util import start,end,file_size
countries, url = common(countries,url,show)
for country in countries:
start("Downloading cache for",country)
file_url = url+"cache."+country.replace(" ","%20")
file_name = download_file(file_url,"cache."+country)
end('')
file_size(file_name)
def download_graph(suffix,countries,url=None,show=False,join=False):
from limic.util import start,end,file_size
countries, url = common(countries,url,show=show,join=join)
for country in countries:
start("Downloading",suffix.upper(),"graph for",country)
file_url = url+"graph."+country.replace(" ","%20")+"."+suffix
file_name = download_file(file_url,"graph."+country+"."+suffix)
end('')
file_size(file_name)
def download_merged(suffixes=("nx","gt","npz"),url=None):
from limic.util import start,end,file_size
if not url:
url = BASE_URL
for suffix in suffixes:
start("Downloading merged",suffix.upper(),"graph for Europe")
file_url = url+"merged.Europe."+suffix
file_name = download_file(file_url,"merged.Europe."+suffix)
end('')
file_size(file_name)
def extract_osm_pre(file_name_in,
file_name_out,
white="{'power':'line'}",
black="{'power':'substation'}",
conserve_mem=False):
from limic.util import start, end, save_pickled, file_size, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
white, black = list(eval(white).items()), list(eval(black).items())
lim = osm_pre(file_name_in, white, black, conserve_mem)
start("Saving data to", file_name_out)
save_pickled(file_name_out, lim)
end('')
file_size(file_name_out)
def convert_npz_nx(file_name_in, file_name_out, penalize=20):
from limic.util import start, end, file_size, save_pickled, load_npz, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading graph from", file_name_in)
g = load_npz(file_name_in)
end('')
file_size(file_name_in)
start("Transforming to NX format")
h = transform_npz_nx(g, penalize)
end()
start("Saving to", file_name_out)
save_pickled(file_name_out, h)
end('')
file_size(file_name_out)
def convert_gt_npz(file_name_in, file_name_out, penalize=20):
from limic.util import start, end, file_size, load_gt, save_npz, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading graph from", file_name_in)
g = load_gt(file_name_in)
end('')
file_size(file_name_in)
start("Initialzing id mapping and neighbours map")
h = transform_gt_npz(g, penalize)
end()
start("Saving to", file_name_out)
save_npz(file_name_out, h)
end('')
file_size(file_name_out)
def convert_nx_gt(file_name_in, file_name_out, rescale=False):
from limic.util import start, end, file_size, load_pickled, save_gt, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading graph from", file_name_in)
g = load_pickled(file_name_in)
end('')
file_size(file_name_in)
start("Initialzing id mapping and neighbours map")
h = transform_nx_gt(g, rescale)
end()
start("Saving to", file_name_out)
save_gt(file_name_out, h)
end('')
file_size(file_name_out)
def merge_cache(file_names, file_name_out):
from limic.util import start, end, file_size, status, load_pickled, save_pickled, check_overwrites
if not check_overwrites(file_names, file_name_out):
return
g = {}
for file_name_in in file_names:
start("Loading cache from", file_name_in)
h = load_pickled(file_name_in)
end('')
file_size(file_name_in)
start("Adding", len(h), "entries")
g.update(h)
end('')
status(len(g))
start("Saving merged cache to", file_name_out)
save_pickled(file_name_out, g)
end('')
file_size(file_name_out)
def extract_osm_post(file_name_in,
file_name_out,
around=1000,
eps=0.01,
safe_dist=100,
penalize=20):
from limic.util import start, end, file_size, load_pickled, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading filtered OSM data from", file_name_in)
lim = load_pickled(file_name_in)
end('')
file_size(file_name_in)
osm_post(lim,
file_name_out,
around=1000,
eps=0.01,
safe_dist=100,
penalize=20)
def shutdown(sig, frame):
nonlocal interrupt
interrupt += 1
print("Shutting down ...")
for f in fs:
f.cancel()
print("Cancelled all futures ...")
running = len(fs)
total = running
while running:
print("Waiting for", running, "processes to shut down ...")
wait(fs, timeout=60)
running = sum(0 if f.done() else 1 for f in fs)
if len_cache != len(region.backend._cache):
file_name_tmp = file_name + "." + str(interrupt)
start("Emergency saving to", file_name_tmp)
save_pickled(file_name_tmp, region.backend._cache)
end('')
file_size(file_name_tmp)
def convert_dbm_cache(file_name_in, file_name_out):
from limic.util import start, end, file_size, status, save_pickled, check_overwrite
from dbm.gnu import open as dopen
from pickle import loads
if not check_overwrite(file_name_in, file_name_out):
return
start("Opening database", file_name_in)
db = dopen(file_name_in, "r")
end('')
file_size(file_name_in)
start("Converting to dictionary")
d = {}
for key in db.keys():
d[key.decode("utf-8")] = loads(db[key])
db.close()
end()
start("Saving to", file_name_out)
save_pickled(file_name_out, d)
end('')
file_size(file_name_out)
def merge_nx(file_names, file_name_out):
from limic.util import start, end, file_size, status, load_pickled, save_pickled, check_overwrites
from networkx import Graph
if not check_overwrites(file_names, file_name_out):
return
g = Graph()
for file_name_in in file_names:
start("Loading graph from", file_name_in)
h = load_pickled(file_name_in)
end('')
file_size(file_name_in)
start("Adding", h.number_of_edges(), "edges")
for from_node, to_node, data in h.edges(data=True):
g.add_edge(from_node, to_node, **data)
end('')
status(g.number_of_edges())
start("Saving merged graph to", file_name_out)
save_pickled(file_name_out, g)
end('')
file_size(file_name_out)
def prune_gt(file_name_in, file_name_out, polygon, overpass_url):
from limic.util import start, end, file_size, status, save_gt, load_gt, check_overwrite
from limic.overpass import nodes_in_geometry, set_server
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading from", file_name_in)
g = load_gt(file_name_in)
end('')
file_size(file_name_in)
start("Query server for nodes in polygon")
set_server(overpass_url)
nodes = nodes_in_geometry(zip(polygon[::2], polygon[1::2]))
end('')
status(len(nodes))
start("Pruning graph")
h = prune_ids_gt(g, nodes)
end()
start("Saving to", file_name_out)
save_gt(file_name_out, h)
end('')
file_size(file_name_out)
def convert_cache_dbm(file_name_in, file_name_out):
from limic.util import start, end, file_size, status, load_pickled, check_overwrite
from dbm.gnu import open as dopen
from os.path import exists
from pickle import dumps
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading from", file_name_in)
d = load_pickled(file_name_in)
end('')
file_size(file_name_in)
start("Opening database", file_name_out)
if not exists(file_name_out):
db = dopen(file_name_out, "c")
db.close()
db = dopen(file_name_out, "c")
end('')
file_size(file_name_out)
start("Computing set of entries to save")
for key in db.keys():
del d[key.decode("utf-8")]
status(len(d))
start("Saving entries to", file_name_out)
for key, val in d.items():
db[key.encode("utf-8")] = dumps(val)
db.close()
end('')
file_size(file_name_out)
def condense(file_name_in, file_name_out, lengths=False, paths=False):
from limic.util import start, end, file_size, status, save_pickled, load_pickled, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading from", file_name_in)
g = load_pickled(file_name_in)
end('')
file_size(file_name_in)
start("Condensing edges")
h = cnx(g)
end()
if lengths:
start("Computing lengths")
h.compute_lengths()
end()
if paths:
start("Computing paths")
h.compute_paths()
end()
start("Saving to", file_name_out)
save_pickled(file_name_out, h)
end('')
file_size(file_name_out)
def fill(overpass_url,
file_name=None,
area=None,
around=1000,
eps=0.01,
safe_dist=100,
penalize=20,
max_workers=None):
from limic.overpass import set_server, pylon, region, get_towers_by_area
from limic.util import start, end, file_size, status, load_pickled, save_pickled, options, replace, options
from networkx import Graph, relabel_nodes
from os import cpu_count
from os.path import exists
from concurrent.futures import ThreadPoolExecutor, wait
from signal import signal, SIGINT
if not area and not file_name:
if options.parser:
options.parser.error("specify at least one of --area or CACHE")
else:
status("ERROR: specify at least area or cache name!")
from sys import exit
exit(-1)
if not area:
area = file_name.split(".")[1]
if not file_name:
file_name = "cache." + area
if not max_workers:
max_workers = cpu_count() * 4
start("Number of workers")
status(max_workers)
if exists(file_name):
start("Loading", file_name)
region.backend._cache = load_pickled(file_name)
end('')
file_size(file_name)
len_cache = len(region.backend._cache)
start("Querying overpass for", area)
set_server(overpass_url)
towers = get_towers_by_area(area)
end()
fs = []
executor = ThreadPoolExecutor(max_workers=max_workers)
interrupt = 0
def shutdown(sig, frame):
nonlocal interrupt
interrupt += 1
print("Shutting down ...")
for f in fs:
f.cancel()
print("Cancelled all futures ...")
running = len(fs)
total = running
while running:
print("Waiting for", running, "processes to shut down ...")
wait(fs, timeout=60)
running = sum(0 if f.done() else 1 for f in fs)
if len_cache != len(region.backend._cache):
file_name_tmp = file_name + "." + str(interrupt)
start("Emergency saving to", file_name_tmp)
save_pickled(file_name_tmp, region.backend._cache)
end('')
file_size(file_name_tmp)
signal(SIGINT, shutdown)
options.failed = True
while options.failed:
options.failed = False
for tower in towers:
fs.append(
executor.submit(cache_tower, tower, around, eps, safe_dist,
penalize))
running = len(fs)
total = running
while running:
print("Waiting for", running, "out of", total, "processes ...")
wait(fs, timeout=60)
running = sum(0 if f.done() else 1 for f in fs)
if len_cache != len(region.backend._cache):
file_name_tmp = file_name + ".tmp"
start("Saving to", file_name, "via", file_name_tmp)
save_pickled(file_name_tmp, region.backend._cache)
replace(file_name_tmp, file_name)
end('')
file_size(file_name)
def render(g,
nodes,
edges,
file_name_out,
markers=False,
lines=False,
host="localhost",
port=5000,
prefix=""):
from limic.util import start, end, status, file_size
from folium import Map, Marker, Icon, PolyLine
from folium.plugins import BeautifyIcon
from binascii import hexlify
from pathlib import Path
from math import log2
from pkg_resources import resource_string
start("Rendering graph")
min_lat = min_long = float('inf')
max_lat = max_long = -float('inf')
for n in nodes:
if n[1] < min_lat: min_lat = n[1]
if n[2] < min_long: min_long = n[2]
if n[1] > max_lat: max_lat = n[1]
if n[2] > max_long: max_long = n[2]
m = Map()
m.fit_bounds([(min_lat, min_long), (max_lat, max_long)])
if markers:
for n in nodes:
Marker(n[1:3],
icon=BeautifyIcon(icon='none',
iconStyle="opacity: 0.1;",
borderColor='#7f7f00',
backgroundColor='#ffff00'),
tooltip=("id: %d" % n[0])).add_to(m)
if lines:
for u, v, weight in edges:
PolyLine([u[1:3], v[1:3]],
color="#3f3f00",
opacity=0.4,
weight=6,
tooltip=("weight: %.1f" % weight)).add_to(m)
TEMPLATE = resource_string("limic", "render.js").decode('utf8')
from branca.element import MacroElement, Template
class LatLngListener(MacroElement):
_template = Template(TEMPLATE % {
'host': host,
'port': port,
'prefix': prefix
})
def __init__(self):
super(MacroElement, self).__init__()
self._name = 'LatLngListener'
LatLngListener().add_to(m)
end()
start("Saving result to HTML file", file_name_out)
#m.save(file_name_out)
end('')
file_size(file_name_out)
return g, nodes
def osm_post(lim,
file_name_out,
around=1000,
eps=0.01,
safe_dist=100,
penalize=20):
from limic.util import start, end, status, file_size, load_pickled, distance, save_pickled
from scipy.spatial import cKDTree as KDTree
from networkx import Graph, astar_path_length
from pyproj import CRS, Transformer
from itertools import chain
from limic.overpass import intersect, pylon
lines, substations, towers, id2tower, id2node, id2lines, id2types = lim
start("Building KD-tree from white nodes")
from limic.util import kdtree
towers_tree = kdtree(towers, get_latlon=lambda x: x.latlon)
end('')
status(len(towers))
start("Deleting black nodes")
to_delete = set()
from limic.util import nodes_in_geometry
for substation in substations:
to_delete.update(
nodes_in_geometry(towers_tree,
list(map(lambda x: id2node[x], substation))))
towers = [tower for tower in towers if tower not in to_delete]
end('')
status(len(towers))
start("Building initial graph")
g = Graph()
g.add_nodes_from(towers)
for line in lines:
line_nodes = list(map(lambda x: id2tower[x], line))
for from_node, to_node in zip(line_nodes, line_nodes[1:]):
if from_node in to_delete or to_node in to_delete:
continue
w = distance(from_node.latlon, to_node.latlon)
g.add_edge(from_node,
to_node,
weight=w,
type=id2types[from_node.id])
end('')
status(len(g.nodes()), end='/')
status(len(g.edges()))
start("Finding neighbours within " + str(around) + "m")
towers_tree = kdtree(towers, get_latlon=lambda x: x.latlon)
end('')
neighbour_indices, neighbours = towers_tree.get_neighbours(around=1000)
end()
start("Computing non-logical intersections")
tower2index = {}
for i, t in zip(range(len(towers)), towers):
tower2index[t] = i
for k, v in id2lines.items():
id2lines[k] = tuple(map(tuple, v))
end('')
segments = set()
for u, v in g.edges():
this = (u, v) if u < v else (v, u)
ui, vi = tower2index[u], tower2index[v]
lines = set()
lines.update(id2lines[u.id])
lines.update(id2lines[v.id])
for neighbour in chain(neighbours[ui], neighbours[vi]):
if neighbour == u or neighbour == v:
continue
if not lines.intersection(id2lines[neighbour.id]):
for nn in g.neighbors(neighbour):
other = (neighbour, nn) if neighbour < nn else (nn,
neighbour)
segments.add(tuple(sorted((this, other))))
end('')
status(len(segments), end=' ')
neighbours2intersection = {}
minusid = 0
latlon2id = {}
segments2intersections = {}
for (t1, t2), (t3, t4) in segments:
res = intersect(t1.latlon,
t2.latlon,
t3.latlon,
t4.latlon,
eps=eps,
no_tu=False)
if res:
intersection, (t, u) = res
if not intersection in latlon2id:
minusid -= 1
latlon2id[intersection] = minusid
segments2intersections.setdefault((t1, t2), []).append(
(t, latlon2id[intersection], intersection))
segments2intersections.setdefault((t3, t4), []).append(
(u, latlon2id[intersection], intersection))
end('')
status(-minusid, end=' ')
for (u, v), intersections in segments2intersections.items():
intersections.sort()
g.remove_edge(u, v)
type = id2types[u.id]
assert (type == id2types[v.id])
seq = [u]
for _, id, latlon in intersections:
seq.append(pylon(id, latlon))
seq.append(v)
for from_node, to_node in zip(seq, seq[1:]):
w = distance(from_node.latlon, to_node.latlon)
g.add_edge(from_node, to_node, weight=w, type=type)
end()
start("Adding routing through air")
airs = set()
for ns in neighbours:
n = ns[0]
for m in ns[1:]:
if not g.has_edge(n, m):
airs.add((n, m))
end('')
for n, m in airs:
w = penalize * distance(n.latlon, m.latlon)
g.add_edge(n, m, weight=w, type=-1)
end('')
status(len(g.nodes()), end='/')
status(len(g.edges()))
from networkx import relabel_nodes
start("Prune redundant edges (incomplete)")
prune_incomplete(g)
end('')
status(len(g.edges()))
start("Prune redundant edges (complete)")
prune_complete(g)
end('')
status(len(g.edges()))
start("Cleaning up graph")
relabel = dict(
map(
lambda tower: (tower,
(tower.id, tower.latlon[0], tower.latlon[1])),
g.nodes()))
relabel_nodes(g, relabel, copy=False)
end()
start("Saving graph to", file_name_out)
save_pickled(file_name_out, g)
end('')
file_size(file_name_out)
from limic.util import start, end, file_size, status, load_pickled, save_pickled
from sys import argv
if __name__ == "__main__":
file_name_in = argv[1]
file_name_out = argv[2]
start("Loading graph from", file_name_in)
g = load_pickled(file_name_in)
end('')
status(len(g.edges()), end=' ')
file_size(file_name_in)
start("Cleaning up graph")
for u, v, d in g.edges(data=True):
d['type'] = -1 if d['air'] else 0
del d['air']
end()
start("Saving graph to", file_name_out)
save_pickled(file_name_out, g)
end('')
file_size(file_name_out)
def extract_cache(file_name_in,
file_name_out,
overpass_url,
area=None,
around=1000,
eps=0.01,
safe_dist=100,
penalize=20):
#from limic.overpass import distance, find_all_neighbours, is_safe, set_server, pylon, region, get_towers_by_area
#from limic.util import start, end, file_size, status, load_pickled, save_pickled, replace, check_overwrite
from networkx import Graph
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading", file_name_in)
region.backend._cache = load_pickled(file_name_in)
len_cache = len(region.backend._cache)
end('')
file_size(file_name_in)
if not area:
area = file_name_in.split(".")[1]
start("Querying overpass for", area)
set_server(overpass_url)
towers = get_towers_by_area(area)
end()
start("Building safe nodes")
g = Graph()
for tower in towers:
if is_safe(tower, safe_dist):
g.add_node(tower)
# else:
# if verbosity >= 2: print("NOT safe!")
end('')
total = len(g.nodes())
status(total)
start("Building edges")
build_edges(g, find_all_neighbours, around, eps, safe_dist, penalize)
end('')
status(len(g.edges()))
if len_cache != len(region.backend._cache):
file_name_tmp = file_name_in + ".tmp"
start("Saving to", file_name_in, "via", file_name_tmp)
save_pickled(file_name_tmp, region.backend._cache)
replace(file_name_tmp, file_name_in)
end('')
file_size(file_name_in)
from limic.util import start, end, status, file_size, save_pickled
from networkx import relabel_nodes
start("Prune redundant edges (incomplete)")
prune_incomplete(g)
end('')
status(len(g.edges()))
start("Prune redundant edges (complete)")
prune_complete(g)
end('')
status(len(g.edges()))
start("Cleaning up graph")
relabel = dict(
map(
lambda tower: (tower,
(tower.id, tower.latlon[0], tower.latlon[1])),
g.nodes()))
relabel_nodes(g, relabel, copy=False)
for u, v, d in g.edges(data=True):
d['type'] = -1 if d['type'] else 0
end()
start("Saving graph to", file_name_out)
save_pickled(file_name_out, g)
end('')
file_size(file_name_out)
