def test_should_add_multiple_nodes_and_find_them():
index = Index(INDEX_BBOX)
index.insert(ITEM1, BBOX1)
index.insert(ITEM2, BBOX2)
assert len(index) == 2
assert index.intersect(BBOX1) == {ITEM1}
assert index.intersect(BBOX2) == {ITEM2}
assert index.intersect(INDEX_BBOX) == {ITEM1, ITEM2}
def test_should_add_multiple_nodes_and_find_them():
index = Index(INDEX_BBOX)
index.insert(ITEM1, BBOX1)
index.insert(ITEM2, BBOX2)
assert len(index) == 2
assert index.intersect(BBOX1) == [ITEM1]
assert index.intersect(BBOX2) == [ITEM2]
res = index.intersect(INDEX_BBOX)
assert ITEM1 in res
assert ITEM2 in res
def remove_overlapping_points(missed_points_coord):
missed_points_qtree = Index(bbox=(np.amin(missed_points_coord[:, 0]), np.amin(missed_points_coord[:,1]), np.amax(missed_points_coord[:,0]), np.amax(missed_points_coord[:, 1])))
d_y = 1/444444.0 ## 0.25 meters
d_x = np.cos(missed_points_coord[0][1] / (180 / np.pi))/444444.0 ## 0.25 meters
for i in trange(len(missed_points_coord), desc='check for double points'):
mp = missed_points_coord[i]
if not missed_points_qtree.intersect((mp[0] - d_x, mp[1] - d_y, mp[0] + d_x, mp[1] + d_y)):
missed_points_qtree.insert(mp, (mp[0] - d_x, mp[1] - d_y, mp[0] + d_x, mp[1] + d_y))
missed_points_coord = missed_points_qtree.intersect(bbox=(np.amin(missed_points_coord[:, 0]), np.amin(missed_points_coord[:,1]), np.amax(missed_points_coord[:,0]), np.amax(missed_points_coord[:, 1])))
return missed_points_coord
def nearest(main_points, aux_points, radius=1):
"""
Objective is to find the nearest point to the points in the aux_points set.
The nearest point is to be identified among the points in the main_points
set.
"""
xmin, ymin = np.min(np.array(list(main_points.values())), axis=0)
xmax, ymax = np.max(np.array(list(main_points.values())), axis=0)
bbox = (xmin, ymin, xmax, ymax)
idx = Index(bbox)
# keep track of points so we can recover them later
points = []
# create bounding box around each point in main_points set
for i, p in enumerate(main_points):
pt_main = Point(main_points[p])
pt_bounds_main = pt_main.x-radius, pt_main.y-radius, \
pt_main.x+radius, pt_main.y+radius
idx.insert(i, pt_bounds_main)
points.append((pt_main, pt_bounds_main, p))
# find intersection with bounding box around aux point set
for n in aux_points:
pt_aux = Point(aux_points[n])
pt_bounds_aux = pt_aux.x-radius, pt_aux.y-radius, \
pt_aux.x+radius, pt_aux.y+radius
matches = idx.intersect(pt_bounds_aux)
ind_closest = min(matches, key=lambda i: geodist(points[i][0], pt_aux))
print(points[ind_closest][-1])
return
def query_quadtree(chr, start, end, files):
chromLength = chromosomes[chr]
print("chromLength", chromLength)
dims = 2
hlevel = math.ceil(math.log2(chromLength) / dims)
print("hlevel", hlevel)
x_y_dim = math.ceil(math.pow(2, hlevel))
print("max x|y =", x_y_dim)
# tree = Index(bbox=(0, 0, x_y_dim, x_y_dim), disk = "quadtree/indexes/roadmap." + chr + ".quadtree.index")
tree = Index(bbox=(0, 0, x_y_dim, x_y_dim),
disk="./roadmap." + chr + ".quadtree.index")
xstart, ystart, _ = hcoords(start, chromLength)
xend, yend, _ = hcoords(end, chromLength)
print("xstart, ystart, xend, yend", xstart, ystart, xend, yend)
margin = 0
overlapbbox = (xstart - margin, ystart - margin, xend + margin,
yend + margin)
matches = tree.intersect(overlapbbox)
df = pandas.DataFrame(matches,
columns=["start", "end", "offset", "size", "fileid"])
print("before file filter")
print(df.shape)
print(df)
df = df[(df["fileid"].isin(files)) & (df["start"] <= end) &
(df["end"] >= start)]
print("after file filter")
print(df.shape)
print(df)
return df
def query(self, file, chr, start, end, zoomlvl=-2):
chromLength = self.genome[chr]
dims = 2
hlevel = math.ceil(math.log2(chromLength) / dims)
# print("hlevel", hlevel)
x_y_dim = math.ceil(math.pow(2, hlevel))
# print("max x|y =", x_y_dim)
tree = Index(bbox=(0, 0, x_y_dim, x_y_dim),
disk=base_path + "quadtree." + chr + ".index")
xstart, ystart, _ = hcoords(start, chromLength)
xend, yend, _ = hcoords(end, chromLength)
overlapbbox = (start - 1, start - 1, end + 1, end + 1)
matches = tree.intersect(overlapbbox)
df = pandas.DataFrame(
matches, columns=["start", "end", "offset", "size", "fileid"])
df = df[df["fileid"] == self.file_mapping[file]]
bw = self.file_objects[file]
chrmId = bw.chrmIds[chr]
result = []
for i, row in df.iterrows():
result += bw.parseLeafDataNode(chrmId, start, end, zoomlvl, chrmId,
row["start"], chrmId, row["end"],
row["offset"], row["size"])
result = toDataFrame(values, bw.columns)
result["chr"] = chr
return result
def produce_fov_ids(lookers,
viewed,
count,
fov_dilation=[-0.1, -0.1, 0.1, 0.1]):
"""
Produce ids of bboxes each of lookers can 'see' (intersect) in the viewed wordbox.
bbox = l, t, r, b
"""
xs = [item[0] for item in viewed] + [item[2] for item in viewed]
ys = [item[1] for item in viewed] + [item[3] for item in viewed]
l = min(xs)
t = min(ys)
r = max(xs)
b = max(ys)
spindex = Index(bbox=(l, t, r, b))
# this example assumes you have a list of items with bbox attribute
for i, bbox in enumerate(viewed):
spindex.insert(i, bbox)
ids_r = np.full((len(lookers), count), -1)
for i, bbox in enumerate(lookers):
matches = spindex.intersect(fov_dilate(bbox, fov_dilation))
# now find the closest to the center of the original:
center = bb_center(bbox)
matches.sort(
key=lambda j: ss_to_center(center, viewed[j])) # ascending
cnt = min(len(matches), count)
ids_r[i, :cnt] = matches[:cnt]
return ids_r
def find_nearest_node(center_cord, node_cord):
"""
Computes the nearest node in the dictionary 'node_cord' to the point denoted
by the 'center_cord'
center_cord:
TYPE: list of two entries
DESCRIPTION: geographical coordinates of the center denoted by a list
of two entries
node_cord:
TYPE: dictionary
DESCRIPTION: dictionary of nodelist with values as the geographical
coordinate
"""
xmin, ymin = np.min(np.array(list(node_cord.values())), axis=0)
xmax, ymax = np.max(np.array(list(node_cord.values())), axis=0)
bbox = (xmin, ymin, xmax, ymax)
idx = Index(bbox)
nodes = []
for i, n in enumerate(list(node_cord.keys())):
node_geom = Point(node_cord[n])
node_bound = bounds(node_geom, 0.0)
idx.insert(i, node_bound)
nodes.append((node_geom, node_bound, n))
pt_center = Point(center_cord)
center_bd = bounds(pt_center, 0.1)
matches = idx.intersect(center_bd)
closest_node = min(matches, key=lambda i: geodist(nodes[i][0], pt_center))
return nodes[closest_node][-1]
def fit_transform(self, X):
self.check_data(X)
max_x, max_y = X.max(axis=0)
quadtree = Index(bbox=[0, 0, max_x + 10, max_y + 10])
for i, x in enumerate(X):
bbox = (x[0], x[1], x[0], x[1])
quadtree.insert(item={
'point': x,
'bbox': bbox,
'id': i
},
bbox=bbox)
maxy = int(max_x / self.size + 50)
maxx = int(max_y / self.size + 50)
cells = []
for i in range(1, maxy + 1):
for j in range(1, maxx + 1):
bbox = ((i - 1) * self.size, (j - 1) * self.size,
i * self.size, j * self.size)
d = quadtree.intersect(bbox)
if len(d) != 0:
cells.append({'x': (i - 1), 'y': (j - 1), 'points': d})
samples = self._sample(quadtree, cells, self.size, self.alpha,
self.size_search, maxx, maxy, 1)
return np.array([u['id'] for u in samples])
def test_should_empty_index_after_removing_multiple_added_nodes_in_all_quad_nodes():
index = Index(INDEX_BBOX, max_items=1)
index.insert(ITEM1, BBOX1)
index.insert(ITEM2, INDEX_BBOX)
index.remove(ITEM1, BBOX1)
index.remove(ITEM2, INDEX_BBOX)
assert len(index) == 0
assert index.intersect(INDEX_BBOX) == []
def combine_osm_components(road,radius = 0.01):
"""
Combines network components by finding nearest nodes
based on a QD Tree Approach.
Parameters
----------
graph : TYPE
DESCRIPTION.
Returns
-------
None.
"""
# Initialize QD Tree
longitudes = [road.nodes[n]['x'] for n in road.nodes()]
latitudes = [road.nodes[n]['y'] for n in road.nodes()]
xmin = min(longitudes); xmax = max(longitudes)
ymin = min(latitudes); ymax = max(latitudes)
bbox = (xmin,ymin,xmax,ymax)
idx = Index(bbox)
# Differentiate large and small components
comps = [c for c in list(nx.connected_components(road))]
lencomps = [len(c) for c in list(nx.connected_components(road))]
indlarge = lencomps.index(max(lencomps))
node_main = list(road.subgraph(comps[indlarge]).nodes())
del(comps[indlarge])
# keep track of nodes so we can recover them later
nodes = []
# create bounding box around each point in large component
for i,node in enumerate(node_main):
pt = Point([road.nodes[node]['x'],road.nodes[node]['y']])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
idx.insert(i, pt_bounds)
nodes.append((pt, pt_bounds, node))
# find intersection and add edges
edgelist = []
for c in comps:
node_comp = list(road.subgraph(c).nodes())
nodepairs = []
for n in node_comp:
pt = Point([road.nodes[n]['x'],road.nodes[n]['y']])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
matches = idx.intersect(pt_bounds)
closest_pt = min(matches,key=lambda i: nodes[i][0].distance(pt))
nodepairs.append((n,nodes[closest_pt][-1]))
# Get the geodesic distance
dist = [MeasureDistance([road.nodes[p[0]]['x'],road.nodes[p[0]]['y']],
[road.nodes[p[1]]['x'],road.nodes[p[1]]['y']]) \
for p in nodepairs]
edgelist.append(nodepairs[np.argmin(dist)]+tuple([0]))
return edgelist
def test_should_empty_index_after_removing_multiple_added_nodes_in_multiple_horizontal_quad_nodes():
index = Index(INDEX_BBOX, max_items=1)
bbox2 = (0, 0, INDEX_BBOX[2], 1)
index.insert(ITEM1, BBOX1)
index.insert(ITEM2, bbox2)
index.remove(ITEM1, BBOX1)
index.remove(ITEM2, bbox2)
assert len(index) == 0
assert index.intersect(INDEX_BBOX) == []
def update(self):
qt = Index(bbox=(0, 0, self.w, self.h), max_items=5)
for i in range(len(self.people)):
s = self.people[i]
qt.insert(i, (s.x, s.y, s.x, s.y))
s.update(self.peopleSpeed)
for s in self.people:
if self.distanceRadius > 0: # 사회적 거리두기
distance = self.distanceRadius
for i in qt.intersect((s.x - distance, s.y - distance,
s.x + distance, s.y + distance)):
other = self.people[i]
if np.sqrt((s.x - other.x) * (s.x - other.x) +
(s.y - other.y) * (s.y - other.y)) > distance:
continue
if s != self.people[i]:
direct = np.arctan2(s.y - self.people[i].y,
s.x - self.people[i].x)
s.dx += np.cos(direct) * 2
s.dy += np.sin(direct) * 2
mag = np.sqrt(s.dx * s.dx + s.dy * s.dy)
s.dx /= mag
s.dy /= mag
for i in qt.intersect(
(s.x - self.infectionRadius, s.y - self.infectionRadius,
s.x + self.infectionRadius, s.y + self.infectionRadius)):
other = self.people[i]
if np.sqrt((s.x - other.x) * (s.x - other.x) +
(s.y - other.y) *
(s.y - other.y)) > self.infectionRadius:
continue
if s.status == 1 and np.random.rand() < self.infectionPercent:
self.people[i].GetInfection(self.recoverTime,
self.reinfectionPercent)
self.setLogData()
def remove_overlapping_points(new_points, missed_points_coord):
d_y = 1/222222.0 ## 0.5 meters
d_x = np.cos(missed_points_coord[0][1] / (180 / np.pi))/222222.0 ## 0.5 meters
missed_new_points_qtree = Index(bbox=(np.amin(new_points[:,0]), np.amin(new_points[:,1]), np.amax(new_points[:,0]), np.amax(new_points[:,1])))
for n in new_points:
if not missed_new_points_qtree.intersect((n[0] - d_x, n[1] - d_y, n[0] + d_x, n[1] + d_y)):
missed_new_points_qtree.insert(n, bbox=(n[0], n[1], n[0], n[1]))
new_points = missed_new_points_qtree.intersect((np.amin(new_points[:,0]), np.amin(new_points[:,1]), np.amax(new_points[:,0]), np.amax(new_points[:,1])))
missed_points_qtree = Index(bbox=(np.amin(missed_points_coord[:, 0]), np.amin(missed_points_coord[:,1]), np.amax(missed_points_coord[:,0]), np.amax(missed_points_coord[:, 1])))
checked_points = []
for p in missed_points_coord:
missed_points_qtree.insert(p, (p[0], p[1], p[0], p[1]))
for i in trange(len(new_points), desc='check for double points'):
mp = new_points[i]
if not missed_points_qtree.intersect((mp[0] - d_x, mp[1] - d_y, mp[0] + d_x, mp[1] + d_y)):
checked_points.append(mp)
return checked_points
def combine_components(graph,cords,radius = 0.01):
"""
Combines network components by finding nearest nodes
based on a QD Tree Approach.
Parameters
----------
graph : TYPE
DESCRIPTION.
Returns
-------
None.
"""
# Initialize QD Tree
xmin,ymin = np.min(np.array(list(cords.values())),axis=0)
xmax,ymax = np.max(np.array(list(cords.values())),axis=0)
bbox = (xmin,ymin,xmax,ymax)
idx = Index(bbox)
# Differentiate large and small components
comps = [c for c in list(nx.connected_components(graph))]
lencomps = [len(c) for c in list(nx.connected_components(graph))]
indlarge = lencomps.index(max(lencomps))
node_main = list(graph.subgraph(comps[indlarge]).nodes())
del(comps[indlarge])
# keep track of lines so we can recover them later
nodes = []
# create bounding box around each point in large component
for i,node in enumerate(node_main):
pt = Point(cords[node])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
idx.insert(i, pt_bounds)
nodes.append((pt, pt_bounds, node))
# find intersection and add edges
edgelist = []
for c in comps:
node_comp = list(graph.subgraph(c).nodes())
nodepairs = []
for n in node_comp:
pt = Point(cords[n])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
matches = idx.intersect(pt_bounds)
closest_pt = min(matches,key=lambda i: nodes[i][0].distance(pt))
nodepairs.append((n,nodes[closest_pt][-1]))
dist = [MeasureDistance(cords[p[0]],cords[p[1]]) for p in nodepairs]
edgelist.append(nodepairs[np.argmin(dist)])
return edgelist
def update(self):
collisionIndex = Index((0, 0, 800, 600))
for o in self.gameObjects:
o.update()
if o.collidable:
collisionIndex.insert(
o, (o.rect.left, o.rect.top, o.rect.right, o.rect.bottom))
for i in self.collidableObjects:
collisions = collisionIndex.intersect(
(i.rect.left, i.rect.top, i.rect.right, i.rect.bottom))
if (len(collisions) > 1): # Intersecting more than self
# logging.info("Collision!")
CollisionHandler(collisions)
class BlockSearch(object):
def __init__(self, blocks):
bboxs = [block.bounding_box for block in blocks]
xmax = max([bb.x + bb.width for bb in bboxs])
ymax = max([bb.y + bb.height for bb in bboxs])
self.spindex = PqtreeIndex(bbox=(0, 0, xmax, ymax))
self.wrapper_map = {}
for block in blocks:
wrapper = DeletableWrapper(block)
self.wrapper_map[block] = wrapper
self.spindex.insert(wrapper, _to_bbox(block.bounding_box))
def find_intersection_with(self, search_bounding_box):
return [
wrapper.data for wrapper in self.spindex.intersect(
_to_bbox(search_bounding_box)) if not wrapper.deleted
]
def remove(self, block):
wrapper = self.wrapper_map.get(block)
if wrapper is not None:
wrapper.deleted = True
class TreeStruct:
bbox = (-10000, -10000, 10000, 10000)
##
# Maintain link from points to rectangle objects?
def __init__(self):
self.index = Index(bbox=self.bbox, max_items=40, max_depth=15)
def addRect(self, rect):
if (bboxoutside(rect.bbox, self.bbox)):
print('outside')
pass # TODO
self.index.insert(rect, rect.bbox)
def removeRect(self, rect):
self.index.remove(rect, rect.bbox)
def query(self, rect):
candidates = self.index.intersect(rect.bbox)
# return candidates
# TBD if we want to make the check both ways or are okay with overlaps only being detected on one end
return [candidate for candidate in candidates if rect.overlaps(candidate) or candidate.overlaps(rect)]
xmin = min(subx)
ymax = max(suby)
ymin = min(suby)
bbox = (xmin, ymin, xmax, ymax)
# Urban substations
idx = Index(bbox)
for pos, poly in enumerate(urban_blocks):
idx.insert(pos, poly.bounds)
#iterate through points
urban_subs = []
for i in range(len(subs)):
point = subs.iloc[i]["geometry"]
# iterate through spatial index
for j in idx.intersect(point.coords[0]):
if point.within(urban_blocks[j]):
urban_subs.append(subs.iloc[i]["ID"])
# Rural substations
idx = Index(bbox)
for pos, poly in enumerate(rural_blocks):
idx.insert(pos, poly.bounds)
#iterate through points
rural_subs = []
for i in range(len(subs)):
point = subs.iloc[i]["geometry"]
# iterate through spatial index
for j in idx.intersect(point.coords[0]):
if point.within(rural_blocks[j]):
pbar0.close()
pyq = Index(bbox=bbox)
for i, p in enumerate(plants):
pyq.insert(i, p.bounds)
convex_hull = Polygon([(p.x, p.y) for p in plants]).convex_hull
pbar = tqdm(total=len(voronoi_paths), position=0)
for j, f in enumerate(polys_list):
pbar1 = tqdm(total=len(polys_list[j]), position=0)
for i, p in enumerate(f):
if (p.is_valid and p.within(convex_hull)) or (
not p.is_valid and p.buffer(0).within(convex_hull)):
bbox = p.bounds
intersected = pyq.intersect(bbox)
if intersected:
for k in intersected:
if not np.isnan(heights_list[j][i]) and abs(
heights_list[j][i]) > 0 and abs(
heights_list[j][i]) < 10:
heights[j][k] = heights_list[j][i] / len(intersected)
pbar1.update(1)
pbar.update(1)
pbar1.close()
pbar.close()
with open(csv_dest, "w") as file:
writer = csv.writer(file,
delimiter=';',
quotechar='"',
# repeating to see overhead
data = pickle.load(open("result1.p", "rb"))
print(len(data))
for entry in data:
print(entry)
(start, end, offset, length, fileName) = (entry[0], entry[1], entry[2],
entry[3], 3)
x, y, _ = hcoords(start, chromLength)
print("x,y", x, y)
tree.insert((start, end, offset, length, fileName), (x, y, x + 1, y + 1))
# need to store the mapping of file names to node ids
map = {
39033: 1,
39031: 2,
39033: 3 # same as 1
}
overlapbbox = (0, 0, 2860, 2860)
matches = tree.intersect(overlapbbox)
print("intersect comes back")
print(len(matches))
# print(matches[0])
print("all matched nodes")
print("format is (start, end, offset, length, fileName)")
for item in matches:
print(item)
print(sys.getsizeof(item))
# print(sys.getsizeof(data))
Polygon(utilv.readable_values_inv(vor.points, mx, my)).bounds)
for i, p in enumerate(utilv.readable_values_inv(vor.points, mx, my)):
spindex.insert({'index': i, 'coord': p}, (p[0], p[1], p[0], p[1]))
pbar15 = tqdm(total=len(ordered), desc="sorting", position=0)
ints = []
for i, key in enumerate(ordered.keys()):
if (isinstance(ordered[key]['boundary'], Polygon)
and ordered[key]['boundary'].exterior) or isinstance(
ordered[key]['boundary'], MultiPolygon):
prepped = prep(ordered[key]['boundary'])
filtered = list(
filter(
lambda a: prepped.contains(
Point(a['coord'][0], a['coord'][1])),
spindex.intersect(ordered[key]['boundary'].bounds)))
points_list[i].extend([p['index'] for p in filtered])
pbar15.update(1)
pbar15.close()
vread_values = np.vectorize(readable_values_inv,
excluded=["mean_x_coord", "mean_y_coord"])
pbar2 = tqdm(total=len(ordered.keys()), desc='sorting more', position=0)
for i, key in enumerate(ordered.keys()):
if (isinstance(ordered[key]['boundary'], Polygon)
and ordered[key]['boundary'].exterior) or isinstance(
ordered[key]['boundary'], MultiPolygon):
prepped = prep(ordered[key]['boundary'])
prepped_exterior = prep(
ordered[key]['boundary'].exterior if isinstance(
ordered[key]['boundary'], Polygon) else unary_union(
class Tiler:
def set_extent(self, tags):
self.max_x = max((tag.x for tag in tags)) + SHIFT
self.min_x = min((tag.x for tag in tags)) - SHIFT
self.max_y = max((tag.y for tag in tags)) + SHIFT
self.min_y = min((tag.y for tag in tags)) - SHIFT
self.origin = Point(self.min_x, self.min_y)
max_size = max(self.max_x - self.min_x, self.max_y - self.min_y)
self.map_size = max_size
self.tile_size = [self.map_size / (1 << i) * METATILE_SIZE for i in range(20)]
self.tag_to_normpos = dict()
for tag in tags:
x, y = tag.x, tag.y
rel_x, rel_y = x - self.origin.x, y - self.origin.y
norm_x, norm_y = rel_x / self.map_size, rel_y / self.map_size
self.tag_to_normpos[tag.name] = (norm_x, norm_y)
def set_bbox(self, tags):
self.tag_spatial_index = Index(bbox=(self.min_x, self.min_y, self.max_x, self.max_y))
for tag in tags:
bbox = (tag.x, tag.y, tag.x, tag.y)
self.tag_spatial_index.insert(tag, bbox)
def set_postcount(self, tags):
for tag in tags:
tag.PostCount = int(getattr(tag, 'PostCount', -1))
self.max_post_count = max(int(tag.PostCount) for tag in tags)
def set_fonts(self):
path_to_font = os.path.join(os.path.dirname(os.path.abspath(__file__)), './Verdana.ttf')
self.fonts = [ImageFont.truetype(path_to_font, ANTIALIASING_SCALE * (25 - zoom * 2)) for zoom in range(8 + 1)]
def __init__(self, tags):
self.set_extent(tags)
self.set_bbox(tags)
self.set_postcount(tags)
self.set_fonts()
def search(self, name):
return self.tag_to_normpos.get(name, '')
def get_postcount_measure(self, tag):
tag_count = tag.PostCount
max_post_count = self.max_post_count
return tag_count / max_post_count
def get_tags_in_tile(self, meta_x, meta_y, zoom, with_shift):
tile_size = self.tile_size[zoom]
lower_left_corner = Point(self.origin.x + meta_x * tile_size,
self.origin.y + meta_y * tile_size)
shift = SHIFT if with_shift else 0
tags_inside_tile = self.tag_spatial_index.intersect((lower_left_corner.x - shift,
lower_left_corner.y - shift,
lower_left_corner.x + tile_size + shift,
lower_left_corner.y + tile_size + shift))
return tags_inside_tile
def get_names_of_shown_tags(self, meta_x, meta_y, zoom):
"""
Return the names of tags that we will show on the map.
On low zoom levels, not all names are shown.
"""
if zoom >= ZOOM_TEXT_SHOW:
# We know that all tag names will be shown anyway, so just return.
return []
tags_inside_tile = self.get_tags_in_tile(meta_x, meta_y, zoom, False)
all_postcounts = sorted([(tag.PostCount, tag.name) for tag in tags_inside_tile])
largest_tags = heapq.nlargest(TAGS_ANNOTATED_PER_TILE, all_postcounts)
return {x[1] for x in largest_tags if x[0] > 0}
def get_metatile(self, meta_x, meta_y, zoom):
'''
Get 8x8 rectangle of tiles, compute them at once.
This is faster than computing them one-by-one.
`meta_x` and `meta_y` are coordinates of upper-left tile of the
generated metatile.
'''
meta_x /= METATILE_SIZE
meta_y /= METATILE_SIZE
img = Image.new('RGB', (TILE_DIM * METATILE_SIZE,
TILE_DIM * METATILE_SIZE), (240, 240, 240))
draw = ImageDraw.Draw(img)
tile_size = self.tile_size[zoom]
lower_left_corner = Point(self.origin.x + meta_x * tile_size,
self.origin.y + meta_y * tile_size)
max_circle_rad = zoom * 1
cnt_points = 0
names_of_shown_tags = set()
for dx in (-1, 0, 1):
for dy in (-1, 0, 1):
names_of_shown_tags.update(self.get_names_of_shown_tags(meta_x + dx, meta_y + dy, zoom))
# Match slightly more tags, so that circles from neighbouring tiles can be drawn partially.
tags_inside_tile = self.get_tags_in_tile(meta_x, meta_y, zoom, True)
def get_point_from_tag(tag):
x, y = tag.x, tag.y
# Get coordinates from tile origin.
point_coords = Point(x - lower_left_corner.x, y - lower_left_corner.y)
# Now scale to TILE_DIM.
pnt = Point(point_coords.x / tile_size * TILE_DIM * METATILE_SIZE,
point_coords.y / tile_size * TILE_DIM * METATILE_SIZE)
return pnt
for tag in tags_inside_tile:
pnt = get_point_from_tag(tag)
# Heuristic formula for showing post counts by circle sizes.
post_count_measure = self.get_postcount_measure(tag)
circle_rad = max(0.5, max_circle_rad * post_count_measure)
draw.ellipse([pnt.x - circle_rad, pnt.y - circle_rad,
pnt.x + circle_rad, pnt.y + circle_rad],
fill=(122, 176, 42))
cnt_points += 1
# Draw text after all circles, so that it is not overwritten.
# (because I did not find any kind of z-index feature in PIL)
for tag in tags_inside_tile:
pnt = get_point_from_tag(tag)
fill = (0, 0, 0)
if zoom >= ZOOM_TEXT_SHOW or tag.name in names_of_shown_tags:
draw.text(pnt, tag.name, fill=fill, font=self.fonts[zoom])
del draw
return img, cnt_points
class MapLink:
"""
This class consists of attributes and methods to evaluate the nearest road
network link to a point. The point may be a home location or a substation.
The algorithm uses a QD-Tree approach to evaluate a bounding box for each
point and link then finds the nearest link to each point.
"""
def __init__(self,road,radius=0.01):
'''
'''
longitudes = [road.nodes[n]['x'] for n in road.nodes()]
latitudes = [road.nodes[n]['y'] for n in road.nodes()]
xmin = min(longitudes); xmax = max(longitudes)
ymin = min(latitudes); ymax = max(latitudes)
bbox = (xmin,ymin,xmax,ymax)
# keep track of lines so we can recover them later
all_link = list(road.edges())
self.lines = []
# initialize the quadtree index
self.idx = Index(bbox)
# add edge bounding boxes to the index
for i, link in enumerate(all_link):
# create line geometry
link_geom = road.edges[link]['geometry']
# bounding boxes, with padding
x1, y1, x2, y2 = link_geom.bounds
bounds = x1-radius, y1-radius, x2+radius, y2+radius
# add to quadtree
self.idx.insert(i, bounds)
# save the line for later use
self.lines.append((link_geom, bounds, link))
return
def map_point(self,points,path=os.getcwd(),fiscode='121',
radius=0.01):
'''
Finds the nearest link to the point under consideration and saves the
map as a csv file in the specified location.
'''
Map2Link = {}
for h in points.cord:
pt = Point(points.cord[h])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
matches = self.idx.intersect(pt_bounds)
# find closest path
try:
closest_path = min(matches,
key=lambda i: self.lines[i][0].distance(pt))
Map2Link[h] = self.lines[closest_path][-1]
except:
Map2Link[h] = None
# Delete unmapped points
unmapped = [p for p in Map2Link if Map2Link[p]==None]
for p in unmapped:
del Map2Link[p]
# Save as a csv file
df_map = pd.DataFrame.from_dict(Map2Link,orient='index',
columns=['source','target'])
df_map.index.names = ['hid']
df_map.to_csv(path+fiscode+'-home2link.csv')
return Map2Link
class MapOSM:
"""
Class consisting of attributes and methods to map OSM links to the residences
"""
def __init__(self,road,radius=0.01):
"""
Initializes the class object by creating a bounding box of known radius
around each OSM road link.
Parameters
----------
road : networkx Multigraph
The Open Street Map multigraph with node and edge attributes.
radius : float, optional
The radius of the bounding box around each road link.
The default is 0.01.
Returns
-------
None.
"""
longitudes = [road.nodes[n]['x'] for n in road.nodes()]
latitudes = [road.nodes[n]['y'] for n in road.nodes()]
xmin = min(longitudes); xmax = max(longitudes)
ymin = min(latitudes); ymax = max(latitudes)
bbox = (xmin,ymin,xmax,ymax)
# keep track of edges so we can recover them later
all_link = list(road.edges(keys=True))
self.links = []
# initialize the quadtree index
self.idx = Index(bbox)
# add edge bounding boxes to the index
for i, link in enumerate(all_link):
# create line geometry
link_geom = road.edges[link]['geometry']
# bounding boxes, with padding
x1, y1, x2, y2 = link_geom.bounds
bounds = x1-radius, y1-radius, x2+radius, y2+radius
# add to quadtree
self.idx.insert(i, bounds)
# save the line for later use
self.links.append((link_geom, bounds, link))
return
def map_point(self,points,path=os.getcwd(),fiscode='121',
radius=0.01):
'''
Finds the nearest link to the residence under consideration and saves the
map as a csv file in the specified location.
'''
Map2Link = {}
for h in points.cord:
pt = Point(points.cord[h])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
matches = self.idx.intersect(pt_bounds)
# find closest path
try:
closest_path = min(matches,
key=lambda i: self.links[i][0].distance(pt))
Map2Link[h] = self.links[closest_path][-1]
except:
Map2Link[h] = None
# Delete unmapped points
unmapped = [p for p in Map2Link if Map2Link[p]==None]
for p in unmapped:
del Map2Link[p]
# Save as a csv file
df_map = pd.DataFrame.from_dict(Map2Link,orient='index',
columns=['source','target','key'])
df_map.index.names = ['hid']
df_map.to_csv(path+fiscode+'-home2OSM.csv')
return Map2Link
def test_should_add_single_node_and_find_its_intersection():
index = Index(INDEX_BBOX)
index.insert(ITEM1, BBOX1)
assert len(index) == 1
assert index.intersect(BBOX1) == [ITEM1]
def test_should_empty_index_after_removing_added_node():
index = Index(INDEX_BBOX)
index.insert(ITEM1, BBOX1)
index.remove(ITEM1, BBOX1)
assert len(index) == 0
assert index.intersect(BBOX1) == []
'index': i,
'plant': list(plants.loc[i, 'geometry'].coords)[0]
}, (list(plants.loc[i, 'geometry'].coords)[0][0],
list(plants.loc[i, 'geometry'].coords)[0][1],
list(plants.loc[i, 'geometry'].coords)[0][0],
list(plants.loc[i, 'geometry'].coords)[0][1]))
pbar = tqdm(total=len(polys_paths), desc="Computing areas", position=0)
for path in polys_paths:
polys = geopandas.read_file(path)
polys = polys.to_crs({'init': 'epsg:28992'})
key = 'area' + re.findall(r'\d+', path)[-1]
pbar2 = tqdm(total=len(polys.index), desc="computing areas", position=0)
for poly in polys.loc[:, 'geometry']:
if poly.is_valid:
plants_i = spindex.intersect(poly.bounds)
if isinstance(plants_i, list):
n = sum([
poly.contains(Point(plant['plant'])) for plant in plants_i
])
for plant in plants_i:
if Point(plant['plant']).within(poly):
plants.loc[plant['index'], key] = (poly.area / n)
else:
n = 1
if Point(plants_i['plant']).within(poly):
plants.loc[plant['index'], key] = (poly.area / n)
pbar2.update(1)
pbar2.close()
pbar.update(1)
pbar.close()
pbar0.close()
pyq = Index(bbox=bbox)
for i, p in enumerate(plants):
pyq.insert(i, p.bounds)
convex_hull = Polygon([(p.x, p.y) for p in plants]).convex_hull
pbar = tqdm(total=len(voronoi_paths), position=0)
for j, f in enumerate(polys_list):
pbar1 = tqdm(total=len(polys_list[j]), position=0)
for p in f:
if (p.is_valid and p.within(convex_hull)) or (
not p.is_valid and p.buffer(0).within(convex_hull)):
bbox = p.bounds
intersected = pyq.intersect(bbox)
if intersected:
for k in intersected:
areas[j][k] = p.area / len(intersected)
pbar1.update(1)
pbar.update(1)
pbar1.close()
pbar.close()
#with open(csv_dest, "w") as file:
# writer = csv.writer(file, delimiter=';', quotechar='"', quoting=csv.QUOTE_MINIMAL)
#
# writer.writerow(['x','y'] + keys)
# for i in range(len(plants)):
# p = plants[i] if isinstance(plants[i], Point) else plants[i][0]
# writer.writerow([p.x, p.y] + [areas[j][i] for j in range(len(keys))])
def filter_and_merge_overlap_boxes(words,
thres,
maxsize,
use_merge_same_line_only=False,
same_line_multiplier=1.0):
if not use_merge_same_line_only:
words_len = [l for _, _, l in words]
words = [(b, m) for b, m, _ in words]
if (len(words) < 2):
if use_merge_same_line_only:
expand_char_mask(words, thres)
return words
# create quad tree
h, w = maxsize
spindex = Index(bbox=(0, 0, w - 1, h - 1))
rects = [Rect((b[0], b[1]), (b[2], b[3])) for b, _ in words]
for i, r in enumerate(rects):
spindex.insert(i, r.as_list())
masks = [m for _, m in words]
is_slant = [False] * len(rects)
for i, mask in enumerate(masks):
mask = (mask > 0)
proj_x = numpy.sum(mask, axis=0)
if (mask.shape[0] > thres * 5 or mask.shape[0] <
thres * 2) and mask.shape[1] > thres * 20 and numpy.percentile(
proj_x, 90) < mask.shape[0] * 0.7:
is_slant[i] = True
else:
masks[i].fill(0)
mh, mw = masks[i].shape
masks[i][int(mh * 0.12):int(mh * 0.88), :].fill(255)
merged_with = [set() for i in range(len(rects))]
done = [False] * len(rects)
new_boxes = []
height_diff_thres = thres
for i in range(len(rects)):
same_line_thres = max(int(
thres * 0.8), 5) if not use_merge_same_line_only else max(
rects[i].height() * 0.3, thres * same_line_multiplier)
overlapbbox = rects[i].pad(same_line_thres, 2).as_list()
matches = spindex.intersect(overlapbbox)
for j in matches:
if (i == j) or is_slant[i] or is_slant[j]: continue
same_line_thres = max(int(
thres * 0.8), 5) if not use_merge_same_line_only else max(
min(rects[i].height(), rects[j].height()) * 0.3, thres *
same_line_multiplier)
#if use_merge_same_line_only: height_diff_thres = max(thres, min(rects[i].height(), rects[j].height()) * 0.7)
rule_intersect_boxes_on_same_line = rects[i].intersectY(rects[j]) > 0.8 * min(rects[i].height(), rects[j].height()) \
and rects[i].intersectX(rects[j], pad = same_line_thres) > 0 \
and rects[i].intersectX(rects[j], pad=0) < thres * 20
rule_big_intersect_area = rects[i].intersectArea(rects[j]) > 0.33 * min(rects[i].area(), rects[j].area()) \
and max(rects[i].width(), rects[j].width()) < thres * 35 \
and min(rects[i].width(), rects[j].width()) < thres * 8
#and min(rects[i].height(), rects[j].height()) > thres * 10
rule_same_height_or_very_short_box = abs(rects[i].height() - rects[j].height()) < height_diff_thres * 0.8 \
or (max(rects[i].height(), rects[j].height()) < thres * 1.8
and min(rects[i].width(), rects[j].width()) < thres * 9) \
or (abs(rects[i].height() - rects[j].height()) < thres * 1.6
and max(rects[i].height(), rects[j].height()) < thres * 3)
rule_small_diacritic_inline = rects[i].intersectY(rects[j]) > 0.94 * rects[i].height() \
and rects[i].intersectX(rects[j], pad=max(int(thres * 0.8), 5)) > 0 and (
rects[i].height() < rects[j].height() * 0.4 and
rects[i].width() < thres * 9) and rects[j].height() < thres * 15 and rects[j].width() > thres * 10
if not use_merge_same_line_only:
rule_same_width_box_intersect = rects[i].intersectX(rects[j], pad=0) > 0.9 * min(rects[i].width(),
rects[j].width()) \
and rects[i].intersectY(rects[j],
pad=2) > 0 and abs(
rects[i].width() - rects[j].width()) < thres \
and max(rects[i].width(), rects[j].width()) < thres * 12 and min(
rects[i].height(), rects[j].height()) < thres * 5 \
and rects[i].height() + rects[j].height() < max(rects[i].width(), rects[j].width()) * 1.3 \
and words_len[i] == 1 and words_len[j] == 1
rule_very_big_intersect_with_height_limit = ((rects[i].intersectY(rects[j]) > 0.6 * max(rects[i].height(), rects[j].height())
and rects[i].intersectX(rects[j], pad = 0) > thres * 1.6)
or (rects[i].intersectX(rects[j]) > 0.4 * max(rects[i].width(), rects[j].width())
and rects[i].intersectY(rects[j]) > thres * 1.4)) and max(rects[i].height(), rects[j].height()) < thres * 3.7 \
and max(rects[i].width(), rects[j].width()) < thres * 30
else:
rule_very_big_intersect_with_height_limit = False
rule_same_width_box_intersect = False
if ((rule_intersect_boxes_on_same_line
or rule_big_intersect_area)
and rule_same_height_or_very_short_box) \
or rule_same_width_box_intersect \
or rule_very_big_intersect_with_height_limit \
or rule_small_diacritic_inline:
merged_with[i].add(j)
merged_with[j].add(i)
#print('merging')
for i in range(len(rects)):
if not done[i]:
new_boxes.append(merge_rect(rects, masks, i, merged_with, done))
### filter overlap boxes:
is_overlap = [False] * len(new_boxes)
spindex = Index(bbox=(0, 0, w - 1, h - 1))
for i in range(len(new_boxes)):
spindex.insert(i, new_boxes[i][0].as_list())
for i in range(len(new_boxes)):
overlapbbox = new_boxes[i][0].pad(3, 3).as_list()
matches = spindex.intersect(overlapbbox)
for j in matches:
if i == j: continue
if new_boxes[i][0].is_overlap_by(new_boxes[j][0]):
is_overlap[i] = True
#print('overlap')
# if new_boxes[j][0].is_overlap_by(new_boxes[i][0]):
# is_overlap[j] = True
# #print('overlap')
if use_merge_same_line_only:
expand_char_mask(new_boxes, thres)
new_boxes = [(new_boxes[i][0].as_list(), new_boxes[i][1])
for i in range(len(new_boxes)) if not is_overlap[i]]
return new_boxes