def process(region):
print region
city = region.split('_')[0]
offset_x, offset_y = int(region.split('_')[1]), int(region.split('_')[2])
sat = scipy.ndimage.imread('{}/{}_sat.png'.format(sat_path, region))
sat = sat.swapaxes(0, 1)
im = scipy.ndimage.imread('{}/{}.png'.format(
out_im_path, region)).astype('float32') / 255.0
im = im.swapaxes(0, 1)
g = graph.read_graph('{}/{}.graph'.format(out_graph_path, region))
g_idx = g.edgeIndex()
gt = graph.read_graph('{}/{}.graph'.format(graph_path, city))
offset = geom.Point(offset_x * 4096, offset_y * 4096)
for vertex in gt.vertices:
vertex.point = vertex.point.sub(offset)
gt_idx = gt.edgeIndex()
connections = get_connections.get_connections(g, im, limit=512)
good, bad = label_gt.label_connections(gt, g, connections)
for i, connection in enumerate(good):
label_gt.write_connection(sat, im, g_idx, connection,
'{}/good/{}_{}'.format(dst_path, region, i))
for i, connection in enumerate(bad):
label_gt.write_connection(sat, im, g_idx, connection,
'{}/bad/{}_{}'.format(dst_path, region, i))
def process(region):
print region
city = region.split('_')[0]
offset_x, offset_y = int(region.split('_')[1]), int(region.split('_')[2])
sat = scipy.ndimage.imread('{}/{}_sat.png'.format(sat_path, region))
sat = sat.swapaxes(0, 1)
im = scipy.ndimage.imread('{}/{}.png'.format(out_im_path, region)).astype('float32') / 255.0
im = im.swapaxes(0, 1)
g = graph.read_graph('{}/{}.graph'.format(out_graph_path, region))
g_idx = g.edgeIndex()
gt = graph.read_graph('{}/{}.graph'.format(graph_path, city))
offset = geom.Point(offset_x * 4096, offset_y * 4096)
for vertex in gt.vertices:
vertex.point = vertex.point.sub(offset)
gt_idx = gt.edgeIndex()
connections = get_connections.get_connections(g, im, limit=512)
good, bad = label_gt.label_connections(gt, g, connections)
for i, connection in enumerate(good):
label_gt.write_connection(sat, im, g_idx, connection, '{}/good/{}_{}'.format(dst_path, region, i))
for i, connection in enumerate(bad):
label_gt.write_connection(sat, im, g_idx, connection, '{}/bad/{}_{}'.format(dst_path, region, i))
def get_gc(self, region):
if region in self.gcs:
return self.gcs[region]
fname = os.path.join(graph_dir, region + '.graph')
g = graph.read_graph(fname)
gc = graph.GraphContainer(g)
self.gcs[region] = gc
return gc
def get_gc(self, region): if region in self.gcs: return self.gcs[region] fname = os.path.join(graph_dir, region + '.graph') g = graph.read_graph(fname) gc = graph.GraphContainer(g) self.gcs[region] = gc return gc
def cleanup_all(graph_fname, im_fname, cleaned_fname):
g = graph.read_graph(graph_fname)
im = numpy.swapaxes(scipy.ndimage.imread(im_fname), 0, 1)
r = geom.Rectangle(geom.Point(0, 0), geom.Point(1300, 1300))
small_r = r.add_tol(-20)
# filter lousy road segments
road_segments, _ = graph.get_graph_road_segments(g)
bad_edges = set()
for rs in road_segments:
if rs.length() < 80 and (len(rs.src().out_edges) < 2 or len(rs.dst().out_edges) < 2) and small_r.contains(rs.src().point) and small_r.contains(rs.dst().point):
bad_edges.update(rs.edges)
elif rs.length() < 400 and len(rs.src().out_edges) < 2 and len(rs.dst().out_edges) < 2 and small_r.contains(rs.src().point) and small_r.contains(rs.dst().point):
bad_edges.update(rs.edges)
ng = graph_filter_edges(g, bad_edges)
# connect road segments to the image edge
road_segments, _ = graph.get_graph_road_segments(ng)
segments = [
geom.Segment(geom.Point(0, 0), geom.Point(1300, 0)),
geom.Segment(geom.Point(0, 0), geom.Point(0, 1300)),
geom.Segment(geom.Point(1300, 1300), geom.Point(1300, 0)),
geom.Segment(geom.Point(1300, 1300), geom.Point(0, 1300)),
]
big_r = r.add_tol(-2)
small_r = r.add_tol(-40)
for rs in road_segments:
for vertex in [rs.src(), rs.dst()]:
if len(vertex.out_edges) == 1 and big_r.contains(vertex.point) and not small_r.contains(vertex.point):
'''d = min([segment.distance(vertex.point) for segment in segments])
dst = get_shortest_path(im, vertex.point.scale(0.5), max_distance=d*9)
if dst is None:
break
if dst is not None:
nv = ng.add_vertex(dst.scale(2))
ng.add_bidirectional_edge(vertex, nv)
print '*** add edge {} to {}'.format(vertex.point, nv.point)'''
'''closest_segment = None
closest_distance = None
for segment in segments:
d = segment.distance(vertex.point)
if closest_segment is None or d < closest_distance:
closest_segment = segment
closest_distance = d'''
for closest_segment in segments:
vector = vertex.in_edges[0].segment().vector()
vector = vector.scale(40.0 / vector.magnitude())
s = geom.Segment(vertex.point, vertex.point.add(vector))
p = s.intersection(closest_segment)
if p is not None:
nv = ng.add_vertex(p)
ng.add_bidirectional_edge(vertex, nv)
break
ng = connect_up(ng, im)
ng.save(cleaned_fname)
def get_gc(self, region):
if region in self.gcs:
return self.gcs[region]
print 'loading gc for {}'.format(region)
fname = os.path.join(graph_dir, region + '.graph')
g = graph.read_graph(fname)
gc = graph.GraphContainer(g)
self.gcs[region] = gc
return gc
def load_tile(k):
print('... load {}'.format(k))
region, x, y = k.split('_')
ims = []
for sat_path in SAT_PATHS:
sat_fname = '{}/{}_sat.jpg'.format(sat_path, k)
im = skimage.io.imread(sat_fname)[:, :, 0:3]
ims.append(im)
g = graph.read_graph('{}/{}.graph'.format(OSM_PATH, k))
road_segments, edge_to_rs = graph.get_graph_road_segments(g)
return (ims, g, road_segments, edge_to_rs, k)
pos2_point = next_positions[0].point()
pos2_pos = next_positions[0]
start_loc = [{
'point': pos1_point,
'edge_pos': pos1_pos,
}, {
'point': pos2_point,
'edge_pos': pos2_pos,
}]
path = model_utils.Path(tile_data['gc'],
tile_data,
start_loc=start_loc)
else:
g = graph.read_graph(EXISTING_GRAPH_FNAME)
r = g.bounds()
tile_data = {
'region': REGION,
'rect': r.add_tol(WINDOW_SIZE / 2),
'search_rect': r,
'cache': cache,
'starting_locations': [],
}
path = model_utils.Path(None, tile_data, g=g)
for vertex in g.vertices:
path.prepend_search_vertex(vertex)
compute_targets = SAVE_EXAMPLES or FOLLOW_TARGETS
if args.e:
ng = graph.read_graph(args.e)
import sys
sys.path.append('../lib')
from discoverlib import graph, geom
BRANCH_THRESHOLD = 15
LOOP_THRESHOLD = 50
in_fname = sys.argv[1]
out_fname = sys.argv[2]
g = graph.read_graph(in_fname)
bad_edges = set()
merge_vertices = {}
merge_groups = []
road_segments, _ = graph.get_graph_road_segments(g)
edge_index = g.edgeIndex()
# prune short branches
for rs in road_segments:
if (len(rs.dst().out_edges) < 2
or len(rs.src().out_edges) < 2) and rs.length() < BRANCH_THRESHOLD:
for edge in rs.edges:
bad_edges.add(edge)
class Group(object):
def __init__(self):
self.l = []
def func(in_fname, out_fname):
g = graph.read_graph(in_fname)
bad_edges = set()
merge_vertices = {}
merge_groups = []
road_segments, _ = graph.get_graph_road_segments(g)
edge_index = g.edgeIndex()
# prune short branches
for rs in road_segments:
if (len(rs.dst().out_edges) < 2 or len(rs.src().out_edges) < 2) and rs.length() < BRANCH_THRESHOLD:
for edge in rs.edges:
bad_edges.add(edge)
# merge short loops
for rs in road_segments:
if rs.length() < LOOP_THRESHOLD:
if rs.src() in merge_vertices and rs.dst() in merge_vertices:
group = merge_vertices[rs.src()]
dst_group = merge_vertices[rs.dst()]
if group != dst_group:
group.update(dst_group)
for vertex in dst_group:
merge_vertices[vertex] = group
elif rs.src() in merge_vertices:
group = merge_vertices[rs.src()]
group.add(rs.dst())
merge_vertices[rs.dst()] = group
elif rs.dst() in merge_vertices:
group = merge_vertices[rs.dst()]
group.add(rs.src())
merge_vertices[rs.src()] = group
else:
group = Group()
group.add(rs.src())
group.add(rs.dst())
merge_vertices[rs.src()] = group
merge_vertices[rs.dst()] = group
merge_groups.append(group)
for edge in rs.edges:
merge_vertices[edge.src] = group
merge_vertices[edge.dst] = group
group.add(edge.src)
group.add(edge.dst)
def get_avg(group):
point_sum = geom.Point(0, 0)
for vertex in group:
point_sum = point_sum.add(vertex.point)
return point_sum.scale(1.0 / len(group))
ng = graph.Graph()
vertex_map = {}
def get_vertex(vertex):
if vertex in merge_vertices:
group = merge_vertices[vertex]
group_head = group.head()
if group_head not in vertex_map:
vertex_map[group_head] = ng.add_vertex(get_avg(group))
return vertex_map[group_head]
else:
if vertex not in vertex_map:
vertex_map[vertex] = ng.add_vertex(vertex.point)
return vertex_map[vertex]
for edge in g.edges:
if edge in bad_edges:
continue
src = get_vertex(edge.src)
dst = get_vertex(edge.dst)
if src == dst:
continue
ng.add_edge(src, dst)
ng.save(out_fname)
def cleanup_all(graph_fname, im_fname, cleaned_fname):
g = graph.read_graph(graph_fname)
im = numpy.swapaxes(scipy.ndimage.imread(im_fname), 0, 1)
r = geom.Rectangle(geom.Point(0, 0), geom.Point(1300, 1300))
small_r = r.add_tol(-20)
# filter lousy road segments
road_segments, _ = graph.get_graph_road_segments(g)
bad_edges = set()
for rs in road_segments:
if rs.length() < 80 and (len(rs.src().out_edges) < 2 or len(
rs.dst().out_edges) < 2) and small_r.contains(
rs.src().point) and small_r.contains(rs.dst().point):
bad_edges.update(rs.edges)
elif rs.length() < 400 and len(rs.src().out_edges) < 2 and len(
rs.dst().out_edges) < 2 and small_r.contains(
rs.src().point) and small_r.contains(rs.dst().point):
bad_edges.update(rs.edges)
ng = graph_filter_edges(g, bad_edges)
# connect road segments to the image edge
road_segments, _ = graph.get_graph_road_segments(ng)
segments = [
geom.Segment(geom.Point(0, 0), geom.Point(1300, 0)),
geom.Segment(geom.Point(0, 0), geom.Point(0, 1300)),
geom.Segment(geom.Point(1300, 1300), geom.Point(1300, 0)),
geom.Segment(geom.Point(1300, 1300), geom.Point(0, 1300)),
]
big_r = r.add_tol(-2)
small_r = r.add_tol(-40)
for rs in road_segments:
for vertex in [rs.src(), rs.dst()]:
if len(vertex.out_edges) == 1 and big_r.contains(
vertex.point) and not small_r.contains(vertex.point):
'''d = min([segment.distance(vertex.point) for segment in segments])
dst = get_shortest_path(im, vertex.point.scale(0.5), max_distance=d*9)
if dst is None:
break
if dst is not None:
nv = ng.add_vertex(dst.scale(2))
ng.add_bidirectional_edge(vertex, nv)
print '*** add edge {} to {}'.format(vertex.point, nv.point)'''
'''closest_segment = None
closest_distance = None
for segment in segments:
d = segment.distance(vertex.point)
if closest_segment is None or d < closest_distance:
closest_segment = segment
closest_distance = d'''
for closest_segment in segments:
vector = vertex.in_edges[0].segment().vector()
vector = vector.scale(40.0 / vector.magnitude())
s = geom.Segment(vertex.point, vertex.point.add(vector))
p = s.intersection(closest_segment)
if p is not None:
nv = ng.add_vertex(p)
ng.add_bidirectional_edge(vertex, nv)
break
ng = connect_up(ng, im)
ng.save(cleaned_fname)
import os
import sys
sys.path.append('../changeseeking_tracing/')
from discoverlib import geom, graph
graph_fname = sys.argv[1]
old_tile_path = sys.argv[2]
new_tile_path = sys.argv[3]
out_path = sys.argv[4]
print('reading graph')
g = graph.read_graph(graph_fname)
print('grid index')
grid_idx = g.edge_grid_index(256)
print('computing subgraphs')
SAT_PATHS = [old_tile_path, new_tile_path]
REGIONS = 'mass'
keys = None
for sat_path in SAT_PATHS:
path_keys = [
fname.split('_sat.jpg')[0] for fname in os.listdir(sat_path)
if '_sat.jpg' in fname
]
if keys is None:
keys = set(path_keys)
else:
keys = keys.intersection(path_keys)
MODEL_PATH = model_path
TEST_PATH = in_path
OUT_PATH = out_path
SAT_PATHS = [old_tile_path, new_tile_path]
VIS_PATH = None
SIZE = 1024
THRESHOLD = 0.1
print('initializing model')
m = model.Model(size=SIZE)
session = tf.Session()
m.saver.restore(session, MODEL_PATH)
print('reading inferred graph')
g = graph.read_graph(TEST_PATH)
print('creating edge index')
idx = g.edgeIndex()
# determine which tiles the graph spans
print('identifying spanned tiles')
tiles = set()
for vertex in g.vertices:
x, y = vertex.point.x / 4096, vertex.point.y / 4096
tiles.add(geom.Point(x, y))
counter = 0
out_graph = graph.Graph()
out_vertex_map = {}
import json
import math
import sys
regions = sys.argv[1].split(',')
fnames = sys.argv[2].split(',')
out_fname = sys.argv[3]
tile_size = 4096.0
json_tiles = []
for i in xrange(len(regions)):
region = regions[i]
fname = fnames[i]
g = graph.read_graph(fname)
tiles = set()
for vertex in g.vertices:
x = int(math.floor(vertex.point.x / tile_size))
y = int(math.floor(vertex.point.y / tile_size))
tiles.add((x, y))
for x, y in tiles:
json_tiles.append({
'x': x,
'y': y,
'region': region,
})
with open(out_fname, 'w') as f:
json.dump(json_tiles, f)
in_dir = sys.argv[1]
threshold = int(sys.argv[2])
out_fname = sys.argv[3]
fnames = os.listdir(in_dir)
fnames = [fname for fname in fnames if fname.endswith('.graph')]
g = graph.Graph()
vertex_map = {}
def get_or_create_vertex(x, y):
if (x, y) not in vertex_map:
vertex_map[(x, y)] = g.add_vertex(geom.Point(x, y))
return vertex_map[(x, y)]
for fname in fnames:
label = fname.split('.')[0]
subg = graph.read_graph(os.path.join(in_dir, fname), merge_duplicates=True, verbose=False)
with open(os.path.join(in_dir, fname.replace('.graph', '.json')), 'r') as f:
edge_probs = json.load(f)
# only use connected components with avg prob exceeding threshold
seen = set()
def dfs(edge, cur):
if edge.id in seen:
return
seen.add(edge.id)
cur.append(edge.id)
for other in edge.src.out_edges:
dfs(other, cur)
for other in edge.dst.out_edges:
dfs(other, cur)
bad = set()
else:
next_positions = graph.follow_graph(pos1_pos, SEGMENT_LENGTH)
pos2_point = next_positions[0].point()
pos2_pos = next_positions[0]
start_loc = [{
'point': pos1_point,
'edge_pos': pos1_pos,
}, {
'point': pos2_point,
'edge_pos': pos2_pos,
}]
path = model_utils.Path(tile_data['gc'], tile_data, start_loc=start_loc)
else:
g = graph.read_graph(EXISTING_GRAPH_FNAME)
r = g.bounds()
tile_data = {
'region': REGION,
'rect': r.add_tol(WINDOW_SIZE/2),
'search_rect': r,
'cache': cache,
'starting_locations': [],
}
path = model_utils.Path(None, tile_data, g=g)
for vertex in g.vertices:
path.prepend_search_vertex(vertex)
compute_targets = SAVE_EXAMPLES or FOLLOW_TARGETS
result = eval([path], m, session, save=SAVE_EXAMPLES, compute_targets=compute_targets, follow_targets=FOLLOW_TARGETS)
print result
max_y = bounds.start.y + padding
return min_x, min_y, max_x, max_y
graph_dir = "/out/graph_infer/c2/"
file_name = os.listdir(graph_dir)
# choose the largest as base graph
large_size = os.path.getsize(graph_dir + file_name[0])
large_id = 0
for i in range(1, len(file_name)):
if os.path.getsize(graph_dir + file_name[i]) > large_size:
large_size = os.path.getsize(graph_dir + file_name[i])
large_id = i
print("the largest graph is {}".format(file_name[large_id]))
graph1 = graph.read_graph(graph_dir + file_name[large_id])
print("base on {}".format(file_name[large_id]))
# # choose the first one as base graph
# graph1 = graph.read_graph(graph_dir + file_name[0])
# print("base on {}".format(file_name[0]))
ind1 = index.Index()
id1 = 0
for edge1 in graph1.edges:
bounds1 = edge1.bounds()
# print(edge1.id)
ind1.insert(
edge1.id,
(bounds1.start.x, bounds1.start.y, bounds1.end.x, bounds1.end.y))
id1 += 1
print("total ids:{}".format(id1))
import sys
sys.path.append('../lib')
from discoverlib import geom, graph
import os
region = sys.argv[1]
in_fname = sys.argv[2]
out_fname = sys.argv[3]
if region == 'boston':
offset = geom.Point(4096, -4096)
elif region == 'chicago':
offset = geom.Point(-4096, -8192)
else:
offset = geom.Point(-4096, -4096)
g = graph.read_graph(in_fname)
for vertex in g.vertices:
vertex.point = vertex.point.add(offset)
g.save(out_fname)
total_not_connected_number = 0
total_pred_number = 0
total_connected_length = 0
total_gt_length = 0
total_pred_length = 0
mylog = open('~/data/out/eval_log/' + log_name + '_connect.log', 'w')
region_name_list = [["amsterdam", -4, -4, 4, 4], ["chicago", -4, -4, 4, 4],
["denver", -4, -4, 4, 4]]
for region_info in region_name_list:
print("test region: " + region_info[0])
graph_name = '~/data/graph_gt/' + region_info[
0] + ".graph" # ground truth graph
gt_graph = graph.read_graph(graph_name)
edge_nodes = []
for i, edge in enumerate(gt_graph.edges):
if i % 2 == 0:
edge_nodes.append([
edge.src.point.x, edge.src.point.y, edge.dst.point.x,
edge.dst.point.y
])
base_gt_mask = np.zeros((1024, 1024))
edge_nodes = np.array(edge_nodes)
for i in range(region_info[1], region_info[3]):
for j in range(region_info[2], region_info[4]):
mask_file = region_info[0] + '_' + str(i) + '_' + str(
j) + '_fusion.png'
# print(mask_dir + mask_file)
# boost_mask_grey_dir = "E:/TGRS/data/cities/out/result_boost2/withDlink_grey/"
# graph_dir = "E:/TGRS/data/cities/out/graph_infer/roadtracer-M/"
save_dir = "E:/TGRS/data/zj/out/result_fusion1/"
if os.path.isdir(save_dir):
pass
else:
os.makedirs(save_dir)
# region_list = ['c', 'g', 'k', 'o']
region_list = ['c2', 'd']
# region_list = ["amsterdam", "chicago", "denver", "la", "montreal", "paris", "pittsburgh", "saltlakecity", "san diego", "tokyo", "toronto", "vancouver"]
for region_name in region_list:
graph_infer = graph.read_graph(graph_dir + region_name + '.out.graph')
edge_nodes = []
for i, edge in enumerate(graph_infer.edges):
if i % 2 == 0:
edge_nodes.append([
edge.src.point.x, edge.src.point.y, edge.dst.point.x,
edge.dst.point.y
])
edge_nodes = np.array(edge_nodes)
for x in range(-4, 4):
for y in range(-4, 4):
boost_mask_grey_file = region_name + '_' + str(x) + '_' + str(
y) + '_boost.png'
boost_mask_grey = cv2.imread(
boost_mask_grey_dir + boost_mask_grey_file, 0) / 255
EXISTING_GRAPH_FNAME = os.path.join(tile_graph_path,
'{}_{}.graph'.format(x, y))
if not os.path.exists(EXISTING_GRAPH_FNAME):
print('skip ({}, {}): no input graph'.format(x, y))
continue
cur_graph_out_fname = os.path.join(out_path,
'{}_{}.graph'.format(x, y))
if os.path.exists(cur_graph_out_fname):
print('skip ({}, {}): already computed outputs'.format(x, y))
continue
r = geom.Rectangle(geom.Point(x * 4096, y * 4096),
geom.Point((x + 1) * 4096, (y + 1) * 4096))
g = graph.read_graph(EXISTING_GRAPH_FNAME)
gc = graph.GraphContainer(g)
g = g.clone()
graph.densify(g, SEGMENT_LENGTH)
tile_data = {
'region': REGION,
'rect': r,
'search_rect': r.add_tol(-WINDOW_SIZE // 2),
'cache': tiles.cache,
'starting_locations': [],
}
#path = model_utils.Path(gc, tile_data, g=g)
path = model_utils.Path(None, tile_data, g=g)
for vertex in g.vertices:
vertex.edge_pos = None
path.prepend_search_vertex(vertex)
import scipy.ndimage
import subprocess
import tensorflow as tf
import time
model_path = sys.argv[1]
sat_path = sys.argv[2]
outim_path = sys.argv[3]
outgraph_path = sys.argv[4]
dst_path = sys.argv[5]
sat = scipy.ndimage.imread(sat_path)
sat = sat.swapaxes(0, 1)
outim = scipy.ndimage.imread(outim_path).astype('float32') / 255.0
outim = outim.swapaxes(0, 1)
g = graph.read_graph(outgraph_path)
g_idx = g.edgeIndex()
print 'loading model'
best_path = model_path + '/model_best/model'
m = model.Model()
session = tf.Session()
m.saver.restore(session, best_path)
print 'getting connections'
connections = get_connections.get_connections(g, outim)
print 'selecting connections'
selected_connections = []
for idx, connection in enumerate(connections):
if idx % 128 == 0:
