def process(region): print region city = region.split('_')[0] offset_x, offset_y = int(region.split('_')[1]), int(region.split('_')[2]) sat = scipy.ndimage.imread('{}/{}_sat.png'.format(sat_path, region)) sat = sat.swapaxes(0, 1) im = scipy.ndimage.imread('{}/{}.png'.format( out_im_path, region)).astype('float32') / 255.0 im = im.swapaxes(0, 1) g = graph.read_graph('{}/{}.graph'.format(out_graph_path, region)) g_idx = g.edgeIndex() gt = graph.read_graph('{}/{}.graph'.format(graph_path, city)) offset = geom.Point(offset_x * 4096, offset_y * 4096) for vertex in gt.vertices: vertex.point = vertex.point.sub(offset) gt_idx = gt.edgeIndex() connections = get_connections.get_connections(g, im, limit=512) good, bad = label_gt.label_connections(gt, g, connections) for i, connection in enumerate(good): label_gt.write_connection(sat, im, g_idx, connection, '{}/good/{}_{}'.format(dst_path, region, i)) for i, connection in enumerate(bad): label_gt.write_connection(sat, im, g_idx, connection, '{}/bad/{}_{}'.format(dst_path, region, i))
def process(region): print region city = region.split('_')[0] offset_x, offset_y = int(region.split('_')[1]), int(region.split('_')[2]) sat = scipy.ndimage.imread('{}/{}_sat.png'.format(sat_path, region)) sat = sat.swapaxes(0, 1) im = scipy.ndimage.imread('{}/{}.png'.format(out_im_path, region)).astype('float32') / 255.0 im = im.swapaxes(0, 1) g = graph.read_graph('{}/{}.graph'.format(out_graph_path, region)) g_idx = g.edgeIndex() gt = graph.read_graph('{}/{}.graph'.format(graph_path, city)) offset = geom.Point(offset_x * 4096, offset_y * 4096) for vertex in gt.vertices: vertex.point = vertex.point.sub(offset) gt_idx = gt.edgeIndex() connections = get_connections.get_connections(g, im, limit=512) good, bad = label_gt.label_connections(gt, g, connections) for i, connection in enumerate(good): label_gt.write_connection(sat, im, g_idx, connection, '{}/good/{}_{}'.format(dst_path, region, i)) for i, connection in enumerate(bad): label_gt.write_connection(sat, im, g_idx, connection, '{}/bad/{}_{}'.format(dst_path, region, i))
def get_gc(self, region): if region in self.gcs: return self.gcs[region] fname = os.path.join(graph_dir, region + '.graph') g = graph.read_graph(fname) gc = graph.GraphContainer(g) self.gcs[region] = gc return gc
def cleanup_all(graph_fname, im_fname, cleaned_fname): g = graph.read_graph(graph_fname) im = numpy.swapaxes(scipy.ndimage.imread(im_fname), 0, 1) r = geom.Rectangle(geom.Point(0, 0), geom.Point(1300, 1300)) small_r = r.add_tol(-20) # filter lousy road segments road_segments, _ = graph.get_graph_road_segments(g) bad_edges = set() for rs in road_segments: if rs.length() < 80 and (len(rs.src().out_edges) < 2 or len(rs.dst().out_edges) < 2) and small_r.contains(rs.src().point) and small_r.contains(rs.dst().point): bad_edges.update(rs.edges) elif rs.length() < 400 and len(rs.src().out_edges) < 2 and len(rs.dst().out_edges) < 2 and small_r.contains(rs.src().point) and small_r.contains(rs.dst().point): bad_edges.update(rs.edges) ng = graph_filter_edges(g, bad_edges) # connect road segments to the image edge road_segments, _ = graph.get_graph_road_segments(ng) segments = [ geom.Segment(geom.Point(0, 0), geom.Point(1300, 0)), geom.Segment(geom.Point(0, 0), geom.Point(0, 1300)), geom.Segment(geom.Point(1300, 1300), geom.Point(1300, 0)), geom.Segment(geom.Point(1300, 1300), geom.Point(0, 1300)), ] big_r = r.add_tol(-2) small_r = r.add_tol(-40) for rs in road_segments: for vertex in [rs.src(), rs.dst()]: if len(vertex.out_edges) == 1 and big_r.contains(vertex.point) and not small_r.contains(vertex.point): '''d = min([segment.distance(vertex.point) for segment in segments]) dst = get_shortest_path(im, vertex.point.scale(0.5), max_distance=d*9) if dst is None: break if dst is not None: nv = ng.add_vertex(dst.scale(2)) ng.add_bidirectional_edge(vertex, nv) print '*** add edge {} to {}'.format(vertex.point, nv.point)''' '''closest_segment = None closest_distance = None for segment in segments: d = segment.distance(vertex.point) if closest_segment is None or d < closest_distance: closest_segment = segment closest_distance = d''' for closest_segment in segments: vector = vertex.in_edges[0].segment().vector() vector = vector.scale(40.0 / vector.magnitude()) s = geom.Segment(vertex.point, vertex.point.add(vector)) p = s.intersection(closest_segment) if p is not None: nv = ng.add_vertex(p) ng.add_bidirectional_edge(vertex, nv) break ng = connect_up(ng, im) ng.save(cleaned_fname)
def get_gc(self, region): if region in self.gcs: return self.gcs[region] print 'loading gc for {}'.format(region) fname = os.path.join(graph_dir, region + '.graph') g = graph.read_graph(fname) gc = graph.GraphContainer(g) self.gcs[region] = gc return gc
def load_tile(k): print('... load {}'.format(k)) region, x, y = k.split('_') ims = [] for sat_path in SAT_PATHS: sat_fname = '{}/{}_sat.jpg'.format(sat_path, k) im = skimage.io.imread(sat_fname)[:, :, 0:3] ims.append(im) g = graph.read_graph('{}/{}.graph'.format(OSM_PATH, k)) road_segments, edge_to_rs = graph.get_graph_road_segments(g) return (ims, g, road_segments, edge_to_rs, k)
pos2_point = next_positions[0].point() pos2_pos = next_positions[0] start_loc = [{ 'point': pos1_point, 'edge_pos': pos1_pos, }, { 'point': pos2_point, 'edge_pos': pos2_pos, }] path = model_utils.Path(tile_data['gc'], tile_data, start_loc=start_loc) else: g = graph.read_graph(EXISTING_GRAPH_FNAME) r = g.bounds() tile_data = { 'region': REGION, 'rect': r.add_tol(WINDOW_SIZE / 2), 'search_rect': r, 'cache': cache, 'starting_locations': [], } path = model_utils.Path(None, tile_data, g=g) for vertex in g.vertices: path.prepend_search_vertex(vertex) compute_targets = SAVE_EXAMPLES or FOLLOW_TARGETS if args.e: ng = graph.read_graph(args.e)
import sys sys.path.append('../lib') from discoverlib import graph, geom BRANCH_THRESHOLD = 15 LOOP_THRESHOLD = 50 in_fname = sys.argv[1] out_fname = sys.argv[2] g = graph.read_graph(in_fname) bad_edges = set() merge_vertices = {} merge_groups = [] road_segments, _ = graph.get_graph_road_segments(g) edge_index = g.edgeIndex() # prune short branches for rs in road_segments: if (len(rs.dst().out_edges) < 2 or len(rs.src().out_edges) < 2) and rs.length() < BRANCH_THRESHOLD: for edge in rs.edges: bad_edges.add(edge) class Group(object): def __init__(self): self.l = []
def func(in_fname, out_fname): g = graph.read_graph(in_fname) bad_edges = set() merge_vertices = {} merge_groups = [] road_segments, _ = graph.get_graph_road_segments(g) edge_index = g.edgeIndex() # prune short branches for rs in road_segments: if (len(rs.dst().out_edges) < 2 or len(rs.src().out_edges) < 2) and rs.length() < BRANCH_THRESHOLD: for edge in rs.edges: bad_edges.add(edge) # merge short loops for rs in road_segments: if rs.length() < LOOP_THRESHOLD: if rs.src() in merge_vertices and rs.dst() in merge_vertices: group = merge_vertices[rs.src()] dst_group = merge_vertices[rs.dst()] if group != dst_group: group.update(dst_group) for vertex in dst_group: merge_vertices[vertex] = group elif rs.src() in merge_vertices: group = merge_vertices[rs.src()] group.add(rs.dst()) merge_vertices[rs.dst()] = group elif rs.dst() in merge_vertices: group = merge_vertices[rs.dst()] group.add(rs.src()) merge_vertices[rs.src()] = group else: group = Group() group.add(rs.src()) group.add(rs.dst()) merge_vertices[rs.src()] = group merge_vertices[rs.dst()] = group merge_groups.append(group) for edge in rs.edges: merge_vertices[edge.src] = group merge_vertices[edge.dst] = group group.add(edge.src) group.add(edge.dst) def get_avg(group): point_sum = geom.Point(0, 0) for vertex in group: point_sum = point_sum.add(vertex.point) return point_sum.scale(1.0 / len(group)) ng = graph.Graph() vertex_map = {} def get_vertex(vertex): if vertex in merge_vertices: group = merge_vertices[vertex] group_head = group.head() if group_head not in vertex_map: vertex_map[group_head] = ng.add_vertex(get_avg(group)) return vertex_map[group_head] else: if vertex not in vertex_map: vertex_map[vertex] = ng.add_vertex(vertex.point) return vertex_map[vertex] for edge in g.edges: if edge in bad_edges: continue src = get_vertex(edge.src) dst = get_vertex(edge.dst) if src == dst: continue ng.add_edge(src, dst) ng.save(out_fname)
def cleanup_all(graph_fname, im_fname, cleaned_fname): g = graph.read_graph(graph_fname) im = numpy.swapaxes(scipy.ndimage.imread(im_fname), 0, 1) r = geom.Rectangle(geom.Point(0, 0), geom.Point(1300, 1300)) small_r = r.add_tol(-20) # filter lousy road segments road_segments, _ = graph.get_graph_road_segments(g) bad_edges = set() for rs in road_segments: if rs.length() < 80 and (len(rs.src().out_edges) < 2 or len( rs.dst().out_edges) < 2) and small_r.contains( rs.src().point) and small_r.contains(rs.dst().point): bad_edges.update(rs.edges) elif rs.length() < 400 and len(rs.src().out_edges) < 2 and len( rs.dst().out_edges) < 2 and small_r.contains( rs.src().point) and small_r.contains(rs.dst().point): bad_edges.update(rs.edges) ng = graph_filter_edges(g, bad_edges) # connect road segments to the image edge road_segments, _ = graph.get_graph_road_segments(ng) segments = [ geom.Segment(geom.Point(0, 0), geom.Point(1300, 0)), geom.Segment(geom.Point(0, 0), geom.Point(0, 1300)), geom.Segment(geom.Point(1300, 1300), geom.Point(1300, 0)), geom.Segment(geom.Point(1300, 1300), geom.Point(0, 1300)), ] big_r = r.add_tol(-2) small_r = r.add_tol(-40) for rs in road_segments: for vertex in [rs.src(), rs.dst()]: if len(vertex.out_edges) == 1 and big_r.contains( vertex.point) and not small_r.contains(vertex.point): '''d = min([segment.distance(vertex.point) for segment in segments]) dst = get_shortest_path(im, vertex.point.scale(0.5), max_distance=d*9) if dst is None: break if dst is not None: nv = ng.add_vertex(dst.scale(2)) ng.add_bidirectional_edge(vertex, nv) print '*** add edge {} to {}'.format(vertex.point, nv.point)''' '''closest_segment = None closest_distance = None for segment in segments: d = segment.distance(vertex.point) if closest_segment is None or d < closest_distance: closest_segment = segment closest_distance = d''' for closest_segment in segments: vector = vertex.in_edges[0].segment().vector() vector = vector.scale(40.0 / vector.magnitude()) s = geom.Segment(vertex.point, vertex.point.add(vector)) p = s.intersection(closest_segment) if p is not None: nv = ng.add_vertex(p) ng.add_bidirectional_edge(vertex, nv) break ng = connect_up(ng, im) ng.save(cleaned_fname)
import os import sys sys.path.append('../changeseeking_tracing/') from discoverlib import geom, graph graph_fname = sys.argv[1] old_tile_path = sys.argv[2] new_tile_path = sys.argv[3] out_path = sys.argv[4] print('reading graph') g = graph.read_graph(graph_fname) print('grid index') grid_idx = g.edge_grid_index(256) print('computing subgraphs') SAT_PATHS = [old_tile_path, new_tile_path] REGIONS = 'mass' keys = None for sat_path in SAT_PATHS: path_keys = [ fname.split('_sat.jpg')[0] for fname in os.listdir(sat_path) if '_sat.jpg' in fname ] if keys is None: keys = set(path_keys) else: keys = keys.intersection(path_keys)
MODEL_PATH = model_path TEST_PATH = in_path OUT_PATH = out_path SAT_PATHS = [old_tile_path, new_tile_path] VIS_PATH = None SIZE = 1024 THRESHOLD = 0.1 print('initializing model') m = model.Model(size=SIZE) session = tf.Session() m.saver.restore(session, MODEL_PATH) print('reading inferred graph') g = graph.read_graph(TEST_PATH) print('creating edge index') idx = g.edgeIndex() # determine which tiles the graph spans print('identifying spanned tiles') tiles = set() for vertex in g.vertices: x, y = vertex.point.x / 4096, vertex.point.y / 4096 tiles.add(geom.Point(x, y)) counter = 0 out_graph = graph.Graph() out_vertex_map = {}
import json import math import sys regions = sys.argv[1].split(',') fnames = sys.argv[2].split(',') out_fname = sys.argv[3] tile_size = 4096.0 json_tiles = [] for i in xrange(len(regions)): region = regions[i] fname = fnames[i] g = graph.read_graph(fname) tiles = set() for vertex in g.vertices: x = int(math.floor(vertex.point.x / tile_size)) y = int(math.floor(vertex.point.y / tile_size)) tiles.add((x, y)) for x, y in tiles: json_tiles.append({ 'x': x, 'y': y, 'region': region, }) with open(out_fname, 'w') as f: json.dump(json_tiles, f)
in_dir = sys.argv[1] threshold = int(sys.argv[2]) out_fname = sys.argv[3] fnames = os.listdir(in_dir) fnames = [fname for fname in fnames if fname.endswith('.graph')] g = graph.Graph() vertex_map = {} def get_or_create_vertex(x, y): if (x, y) not in vertex_map: vertex_map[(x, y)] = g.add_vertex(geom.Point(x, y)) return vertex_map[(x, y)] for fname in fnames: label = fname.split('.')[0] subg = graph.read_graph(os.path.join(in_dir, fname), merge_duplicates=True, verbose=False) with open(os.path.join(in_dir, fname.replace('.graph', '.json')), 'r') as f: edge_probs = json.load(f) # only use connected components with avg prob exceeding threshold seen = set() def dfs(edge, cur): if edge.id in seen: return seen.add(edge.id) cur.append(edge.id) for other in edge.src.out_edges: dfs(other, cur) for other in edge.dst.out_edges: dfs(other, cur) bad = set()
else: next_positions = graph.follow_graph(pos1_pos, SEGMENT_LENGTH) pos2_point = next_positions[0].point() pos2_pos = next_positions[0] start_loc = [{ 'point': pos1_point, 'edge_pos': pos1_pos, }, { 'point': pos2_point, 'edge_pos': pos2_pos, }] path = model_utils.Path(tile_data['gc'], tile_data, start_loc=start_loc) else: g = graph.read_graph(EXISTING_GRAPH_FNAME) r = g.bounds() tile_data = { 'region': REGION, 'rect': r.add_tol(WINDOW_SIZE/2), 'search_rect': r, 'cache': cache, 'starting_locations': [], } path = model_utils.Path(None, tile_data, g=g) for vertex in g.vertices: path.prepend_search_vertex(vertex) compute_targets = SAVE_EXAMPLES or FOLLOW_TARGETS result = eval([path], m, session, save=SAVE_EXAMPLES, compute_targets=compute_targets, follow_targets=FOLLOW_TARGETS) print result
max_y = bounds.start.y + padding return min_x, min_y, max_x, max_y graph_dir = "/out/graph_infer/c2/" file_name = os.listdir(graph_dir) # choose the largest as base graph large_size = os.path.getsize(graph_dir + file_name[0]) large_id = 0 for i in range(1, len(file_name)): if os.path.getsize(graph_dir + file_name[i]) > large_size: large_size = os.path.getsize(graph_dir + file_name[i]) large_id = i print("the largest graph is {}".format(file_name[large_id])) graph1 = graph.read_graph(graph_dir + file_name[large_id]) print("base on {}".format(file_name[large_id])) # # choose the first one as base graph # graph1 = graph.read_graph(graph_dir + file_name[0]) # print("base on {}".format(file_name[0])) ind1 = index.Index() id1 = 0 for edge1 in graph1.edges: bounds1 = edge1.bounds() # print(edge1.id) ind1.insert( edge1.id, (bounds1.start.x, bounds1.start.y, bounds1.end.x, bounds1.end.y)) id1 += 1 print("total ids:{}".format(id1))
import sys sys.path.append('../lib') from discoverlib import geom, graph import os region = sys.argv[1] in_fname = sys.argv[2] out_fname = sys.argv[3] if region == 'boston': offset = geom.Point(4096, -4096) elif region == 'chicago': offset = geom.Point(-4096, -8192) else: offset = geom.Point(-4096, -4096) g = graph.read_graph(in_fname) for vertex in g.vertices: vertex.point = vertex.point.add(offset) g.save(out_fname)
total_not_connected_number = 0 total_pred_number = 0 total_connected_length = 0 total_gt_length = 0 total_pred_length = 0 mylog = open('~/data/out/eval_log/' + log_name + '_connect.log', 'w') region_name_list = [["amsterdam", -4, -4, 4, 4], ["chicago", -4, -4, 4, 4], ["denver", -4, -4, 4, 4]] for region_info in region_name_list: print("test region: " + region_info[0]) graph_name = '~/data/graph_gt/' + region_info[ 0] + ".graph" # ground truth graph gt_graph = graph.read_graph(graph_name) edge_nodes = [] for i, edge in enumerate(gt_graph.edges): if i % 2 == 0: edge_nodes.append([ edge.src.point.x, edge.src.point.y, edge.dst.point.x, edge.dst.point.y ]) base_gt_mask = np.zeros((1024, 1024)) edge_nodes = np.array(edge_nodes) for i in range(region_info[1], region_info[3]): for j in range(region_info[2], region_info[4]): mask_file = region_info[0] + '_' + str(i) + '_' + str( j) + '_fusion.png' # print(mask_dir + mask_file)
# boost_mask_grey_dir = "E:/TGRS/data/cities/out/result_boost2/withDlink_grey/" # graph_dir = "E:/TGRS/data/cities/out/graph_infer/roadtracer-M/" save_dir = "E:/TGRS/data/zj/out/result_fusion1/" if os.path.isdir(save_dir): pass else: os.makedirs(save_dir) # region_list = ['c', 'g', 'k', 'o'] region_list = ['c2', 'd'] # region_list = ["amsterdam", "chicago", "denver", "la", "montreal", "paris", "pittsburgh", "saltlakecity", "san diego", "tokyo", "toronto", "vancouver"] for region_name in region_list: graph_infer = graph.read_graph(graph_dir + region_name + '.out.graph') edge_nodes = [] for i, edge in enumerate(graph_infer.edges): if i % 2 == 0: edge_nodes.append([ edge.src.point.x, edge.src.point.y, edge.dst.point.x, edge.dst.point.y ]) edge_nodes = np.array(edge_nodes) for x in range(-4, 4): for y in range(-4, 4): boost_mask_grey_file = region_name + '_' + str(x) + '_' + str( y) + '_boost.png' boost_mask_grey = cv2.imread( boost_mask_grey_dir + boost_mask_grey_file, 0) / 255
EXISTING_GRAPH_FNAME = os.path.join(tile_graph_path, '{}_{}.graph'.format(x, y)) if not os.path.exists(EXISTING_GRAPH_FNAME): print('skip ({}, {}): no input graph'.format(x, y)) continue cur_graph_out_fname = os.path.join(out_path, '{}_{}.graph'.format(x, y)) if os.path.exists(cur_graph_out_fname): print('skip ({}, {}): already computed outputs'.format(x, y)) continue r = geom.Rectangle(geom.Point(x * 4096, y * 4096), geom.Point((x + 1) * 4096, (y + 1) * 4096)) g = graph.read_graph(EXISTING_GRAPH_FNAME) gc = graph.GraphContainer(g) g = g.clone() graph.densify(g, SEGMENT_LENGTH) tile_data = { 'region': REGION, 'rect': r, 'search_rect': r.add_tol(-WINDOW_SIZE // 2), 'cache': tiles.cache, 'starting_locations': [], } #path = model_utils.Path(gc, tile_data, g=g) path = model_utils.Path(None, tile_data, g=g) for vertex in g.vertices: vertex.edge_pos = None path.prepend_search_vertex(vertex)
import scipy.ndimage import subprocess import tensorflow as tf import time model_path = sys.argv[1] sat_path = sys.argv[2] outim_path = sys.argv[3] outgraph_path = sys.argv[4] dst_path = sys.argv[5] sat = scipy.ndimage.imread(sat_path) sat = sat.swapaxes(0, 1) outim = scipy.ndimage.imread(outim_path).astype('float32') / 255.0 outim = outim.swapaxes(0, 1) g = graph.read_graph(outgraph_path) g_idx = g.edgeIndex() print 'loading model' best_path = model_path + '/model_best/model' m = model.Model() session = tf.Session() m.saver.restore(session, best_path) print 'getting connections' connections = get_connections.get_connections(g, outim) print 'selecting connections' selected_connections = [] for idx, connection in enumerate(connections): if idx % 128 == 0: