def convert_nx_npz(file_name_in,
file_name_out,
indirect=False,
rescale=False,
penalize=20):
from limic.util import start, end, file_size, load_pickled, save_npz, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading graph from", file_name_in)
g = load_pickled(file_name_in)
end('')
file_size(file_name_in)
if indirect:
start("Transforming to GT format")
i = transform_nx_gt(g, rescale)
end()
start("Transforming to NPZ format")
h = transform_gt_npz(i, penalize)
end()
else:
start("Transforming to NPZ format")
h = transform_nx_npz(g, rescale)
end()
start("Saving to", file_name_out)
save_npz(file_name_out, h)
end('')
file_size(file_name_out)
def render_npz(file_name_in,
file_name_out,
markers=False,
lines=False,
host="localhost",
port=5000,
prefix=""):
from limic.util import start, end, load_npz, check_overwrite
from numpy import column_stack
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading NPZ graph", file_name_in)
g = load_npz(file_name_in)
nodes = list(map(tuple, column_stack((g['ids'], g['lat'], g['long']))))
edges = []
if lines:
id2edges = g['id2edges']
edges_weight = g['edges_weight']
edges_neighbor = g['edges_neighbor']
for index in range(len(nodes)):
left = id2edges[index]
right = id2edges[index + 1]
me = nodes[index]
for i in range(left, right):
edges.append((me, nodes[edges_neighbor[i]], edges_weight[i]))
end()
return render(g, nodes, edges, file_name_out, markers, lines, host, port,
prefix)
def prune_nx(file_name_in, file_name_out, polygon, overpass_url):
from limic.util import start, end, file_size, status, save_pickled, load_pickled, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading from", file_name_in)
g = load_pickled(file_name_in)
end('')
file_size(file_name_in)
polygon = list(map(float, polygon))
polygon = list(zip(polygon[::2], polygon[1::2]))
if not overpass_url:
from limic.util import kdtree, nodes_in_geometry
start("Building kd-tree from nodes")
tree = kdtree(g.nodes(), get_latlon=lambda x: (x[1], x[2]))
end()
start("Querying tree for nodes in polygon")
nodes = nodes_in_geometry(tree, polygon)
else:
from limic.overpass import nodes_in_geometry, set_server
start("Query server for nodes in polygon")
set_server(overpass_url)
nodes = nodes_in_geometry(polygon)
end('')
status(len(nodes))
start("Pruning graph")
h = prune_ids_nx(g, nodes)
end()
start("Saving to", file_name_out)
save_pickled(file_name_out, h)
end('')
file_size(file_name_out)
def convert_cache_dbm(file_name_in, file_name_out):
from limic.util import start, end, file_size, status, load_pickled, check_overwrite
from dbm.gnu import open as dopen
from os.path import exists
from pickle import dumps
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading from", file_name_in)
d = load_pickled(file_name_in)
end('')
file_size(file_name_in)
start("Opening database", file_name_out)
if not exists(file_name_out):
db = dopen(file_name_out, "c")
db.close()
db = dopen(file_name_out, "c")
end('')
file_size(file_name_out)
start("Computing set of entries to save")
for key in db.keys():
del d[key.decode("utf-8")]
status(len(d))
start("Saving entries to", file_name_out)
for key, val in d.items():
db[key.encode("utf-8")] = dumps(val)
db.close()
end('')
file_size(file_name_out)
def extract_osm_pre(file_name_in,
file_name_out,
white="{'power':'line'}",
black="{'power':'substation'}",
conserve_mem=False):
from limic.util import start, end, save_pickled, file_size, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
white, black = list(eval(white).items()), list(eval(black).items())
lim = osm_pre(file_name_in, white, black, conserve_mem)
start("Saving data to", file_name_out)
save_pickled(file_name_out, lim)
end('')
file_size(file_name_out)
def convert_npz_nx(file_name_in, file_name_out, penalize=20):
from limic.util import start, end, file_size, save_pickled, load_npz, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading graph from", file_name_in)
g = load_npz(file_name_in)
end('')
file_size(file_name_in)
start("Transforming to NX format")
h = transform_npz_nx(g, penalize)
end()
start("Saving to", file_name_out)
save_pickled(file_name_out, h)
end('')
file_size(file_name_out)
def convert_gt_npz(file_name_in, file_name_out, penalize=20):
from limic.util import start, end, file_size, load_gt, save_npz, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading graph from", file_name_in)
g = load_gt(file_name_in)
end('')
file_size(file_name_in)
start("Initialzing id mapping and neighbours map")
h = transform_gt_npz(g, penalize)
end()
start("Saving to", file_name_out)
save_npz(file_name_out, h)
end('')
file_size(file_name_out)
def convert_nx_gt(file_name_in, file_name_out, rescale=False):
from limic.util import start, end, file_size, load_pickled, save_gt, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading graph from", file_name_in)
g = load_pickled(file_name_in)
end('')
file_size(file_name_in)
start("Initialzing id mapping and neighbours map")
h = transform_nx_gt(g, rescale)
end()
start("Saving to", file_name_out)
save_gt(file_name_out, h)
end('')
file_size(file_name_out)
def render_nx(file_name_in,
file_name_out,
markers=False,
lines=False,
host="localhost",
port=5000,
prefix=""):
from limic.util import start, end, load_pickled, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading NX graph", file_name_in)
g = load_pickled(file_name_in)
nodes = list(g.nodes())
edges = list(g.edges.data('weight'))
end()
return render(g, nodes, edges, file_name_out, markers, lines, host, port,
prefix)
def extract_osm_post(file_name_in,
file_name_out,
around=1000,
eps=0.01,
safe_dist=100,
penalize=20):
from limic.util import start, end, file_size, load_pickled, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading filtered OSM data from", file_name_in)
lim = load_pickled(file_name_in)
end('')
file_size(file_name_in)
osm_post(lim,
file_name_out,
around=1000,
eps=0.01,
safe_dist=100,
penalize=20)
def render_cnx(file_name_in,
file_name_out,
components=[],
markers=False,
lines=False,
host="localhost",
port=5000,
prefix=""):
from limic.util import start, end, load_pickled, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading CNX graph", file_name_in)
cs, _, _ = load_pickled(file_name_in)
end()
for i in components if components else range(len(cs)):
nodes = list(cs[i].nodes())
edges = list(cs[i].edges.data('weight'))
file_out = file_name_out.split(".html")[0] + "." + str(i) + ".html"
render(cs[i], nodes, edges, file_out, markers, lines, host, port,
prefix)
def convert_dbm_cache(file_name_in, file_name_out):
from limic.util import start, end, file_size, status, save_pickled, check_overwrite
from dbm.gnu import open as dopen
from pickle import loads
if not check_overwrite(file_name_in, file_name_out):
return
start("Opening database", file_name_in)
db = dopen(file_name_in, "r")
end('')
file_size(file_name_in)
start("Converting to dictionary")
d = {}
for key in db.keys():
d[key.decode("utf-8")] = loads(db[key])
db.close()
end()
start("Saving to", file_name_out)
save_pickled(file_name_out, d)
end('')
file_size(file_name_out)
def prune_gt(file_name_in, file_name_out, polygon, overpass_url):
from limic.util import start, end, file_size, status, save_gt, load_gt, check_overwrite
from limic.overpass import nodes_in_geometry, set_server
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading from", file_name_in)
g = load_gt(file_name_in)
end('')
file_size(file_name_in)
start("Query server for nodes in polygon")
set_server(overpass_url)
nodes = nodes_in_geometry(zip(polygon[::2], polygon[1::2]))
end('')
status(len(nodes))
start("Pruning graph")
h = prune_ids_gt(g, nodes)
end()
start("Saving to", file_name_out)
save_gt(file_name_out, h)
end('')
file_size(file_name_out)
def extract_osm(file_name_in,
file_name_out,
around=1000,
eps=0.01,
safe_dist=100,
penalize=20,
white="{'power':'line'}",
black="{'power':'substation'}",
conserve_mem=False):
from setproctitle import setproctitle
setproctitle(file_name_in)
from limic.util import check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
white, black = list(eval(white).items()), list(eval(black).items())
lim = osm_pre(file_name_in, white, black, conserve_mem)
osm_post(lim,
file_name_out,
around=1000,
eps=0.01,
safe_dist=100,
penalize=20)
def condense(file_name_in, file_name_out, lengths=False, paths=False):
from limic.util import start, end, file_size, status, save_pickled, load_pickled, check_overwrite
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading from", file_name_in)
g = load_pickled(file_name_in)
end('')
file_size(file_name_in)
start("Condensing edges")
h = cnx(g)
end()
if lengths:
start("Computing lengths")
h.compute_lengths()
end()
if paths:
start("Computing paths")
h.compute_paths()
end()
start("Saving to", file_name_out)
save_pickled(file_name_out, h)
end('')
file_size(file_name_out)
def extract_cache(file_name_in,
file_name_out,
overpass_url,
area=None,
around=1000,
eps=0.01,
safe_dist=100,
penalize=20):
#from limic.overpass import distance, find_all_neighbours, is_safe, set_server, pylon, region, get_towers_by_area
#from limic.util import start, end, file_size, status, load_pickled, save_pickled, replace, check_overwrite
from networkx import Graph
if not check_overwrite(file_name_in, file_name_out):
return
start("Loading", file_name_in)
region.backend._cache = load_pickled(file_name_in)
len_cache = len(region.backend._cache)
end('')
file_size(file_name_in)
if not area:
area = file_name_in.split(".")[1]
start("Querying overpass for", area)
set_server(overpass_url)
towers = get_towers_by_area(area)
end()
start("Building safe nodes")
g = Graph()
for tower in towers:
if is_safe(tower, safe_dist):
g.add_node(tower)
# else:
# if verbosity >= 2: print("NOT safe!")
end('')
total = len(g.nodes())
status(total)
start("Building edges")
build_edges(g, find_all_neighbours, around, eps, safe_dist, penalize)
end('')
status(len(g.edges()))
if len_cache != len(region.backend._cache):
file_name_tmp = file_name_in + ".tmp"
start("Saving to", file_name_in, "via", file_name_tmp)
save_pickled(file_name_tmp, region.backend._cache)
replace(file_name_tmp, file_name_in)
end('')
file_size(file_name_in)
from limic.util import start, end, status, file_size, save_pickled
from networkx import relabel_nodes
start("Prune redundant edges (incomplete)")
prune_incomplete(g)
end('')
status(len(g.edges()))
start("Prune redundant edges (complete)")
prune_complete(g)
end('')
status(len(g.edges()))
start("Cleaning up graph")
relabel = dict(
map(
lambda tower: (tower,
(tower.id, tower.latlon[0], tower.latlon[1])),
g.nodes()))
relabel_nodes(g, relabel, copy=False)
for u, v, d in g.edges(data=True):
d['type'] = -1 if d['type'] else 0
end()
start("Saving graph to", file_name_out)
save_pickled(file_name_out, g)
end('')
file_size(file_name_out)
