def add_to_index(self, file, zoomlvl=-2):
tree, bw = self.get_file_btree(file, zoomlvl)
df = self.get_leaf_nodes(tree, bw, zoomlvl)
chrmTree = self.get_file_chr(bw)
self.file_mapping[file] = self.file_counter
self.file_counter += 1
self.file_objects[file] = bw
for chrm in chrmTree.keys():
chromLength = self.genome[chrm]
dims = 2
hlevel = math.ceil(math.log2(chromLength) / dims)
# print("hlevel", hlevel)
x_y_dim = math.ceil(math.pow(2, hlevel))
# print("max x|y =", x_y_dim)
tree = Index(bbox=(0, 0, x_y_dim, x_y_dim),
disk=base_path + "quadtree." + chrm + ".index")
chrmId = chrmTree[chrm]
df = df[df["rStartChromIx"] == chrmId]
# print("\t df shape - ", df.shape)
for i, row in df.iterrows():
x, y, _ = hcoords(row["rStartBase"], chromLength)
tree.insert(
(row["rStartBase"], row["rEndBase"], row["rdataOffset"],
row["rDataSize"], fileIds[file]), (x, y, x + 1, y + 1))
def fit_transform(self, X):
self.check_data(X)
max_x, max_y = X.max(axis=0)
quadtree = Index(bbox=[0, 0, max_x + 10, max_y + 10])
for i, x in enumerate(X):
bbox = (x[0], x[1], x[0], x[1])
quadtree.insert(item={
'point': x,
'bbox': bbox,
'id': i
},
bbox=bbox)
maxy = int(max_x / self.size + 50)
maxx = int(max_y / self.size + 50)
cells = []
for i in range(1, maxy + 1):
for j in range(1, maxx + 1):
bbox = ((i - 1) * self.size, (j - 1) * self.size,
i * self.size, j * self.size)
d = quadtree.intersect(bbox)
if len(d) != 0:
cells.append({'x': (i - 1), 'y': (j - 1), 'points': d})
samples = self._sample(quadtree, cells, self.size, self.alpha,
self.size_search, maxx, maxy, 1)
return np.array([u['id'] for u in samples])
def nearest(main_points, aux_points, radius=1):
"""
Objective is to find the nearest point to the points in the aux_points set.
The nearest point is to be identified among the points in the main_points
set.
"""
xmin, ymin = np.min(np.array(list(main_points.values())), axis=0)
xmax, ymax = np.max(np.array(list(main_points.values())), axis=0)
bbox = (xmin, ymin, xmax, ymax)
idx = Index(bbox)
# keep track of points so we can recover them later
points = []
# create bounding box around each point in main_points set
for i, p in enumerate(main_points):
pt_main = Point(main_points[p])
pt_bounds_main = pt_main.x-radius, pt_main.y-radius, \
pt_main.x+radius, pt_main.y+radius
idx.insert(i, pt_bounds_main)
points.append((pt_main, pt_bounds_main, p))
# find intersection with bounding box around aux point set
for n in aux_points:
pt_aux = Point(aux_points[n])
pt_bounds_aux = pt_aux.x-radius, pt_aux.y-radius, \
pt_aux.x+radius, pt_aux.y+radius
matches = idx.intersect(pt_bounds_aux)
ind_closest = min(matches, key=lambda i: geodist(points[i][0], pt_aux))
print(points[ind_closest][-1])
return
def find_nearest_node(center_cord, node_cord):
"""
Computes the nearest node in the dictionary 'node_cord' to the point denoted
by the 'center_cord'
center_cord:
TYPE: list of two entries
DESCRIPTION: geographical coordinates of the center denoted by a list
of two entries
node_cord:
TYPE: dictionary
DESCRIPTION: dictionary of nodelist with values as the geographical
coordinate
"""
xmin, ymin = np.min(np.array(list(node_cord.values())), axis=0)
xmax, ymax = np.max(np.array(list(node_cord.values())), axis=0)
bbox = (xmin, ymin, xmax, ymax)
idx = Index(bbox)
nodes = []
for i, n in enumerate(list(node_cord.keys())):
node_geom = Point(node_cord[n])
node_bound = bounds(node_geom, 0.0)
idx.insert(i, node_bound)
nodes.append((node_geom, node_bound, n))
pt_center = Point(center_cord)
center_bd = bounds(pt_center, 0.1)
matches = idx.intersect(center_bd)
closest_node = min(matches, key=lambda i: geodist(nodes[i][0], pt_center))
return nodes[closest_node][-1]
def produce_fov_ids(lookers,
viewed,
count,
fov_dilation=[-0.1, -0.1, 0.1, 0.1]):
"""
Produce ids of bboxes each of lookers can 'see' (intersect) in the viewed wordbox.
bbox = l, t, r, b
"""
xs = [item[0] for item in viewed] + [item[2] for item in viewed]
ys = [item[1] for item in viewed] + [item[3] for item in viewed]
l = min(xs)
t = min(ys)
r = max(xs)
b = max(ys)
spindex = Index(bbox=(l, t, r, b))
# this example assumes you have a list of items with bbox attribute
for i, bbox in enumerate(viewed):
spindex.insert(i, bbox)
ids_r = np.full((len(lookers), count), -1)
for i, bbox in enumerate(lookers):
matches = spindex.intersect(fov_dilate(bbox, fov_dilation))
# now find the closest to the center of the original:
center = bb_center(bbox)
matches.sort(
key=lambda j: ss_to_center(center, viewed[j])) # ascending
cnt = min(len(matches), count)
ids_r[i, :cnt] = matches[:cnt]
return ids_r
def test_should_empty_index_after_removing_multiple_added_nodes_in_all_quad_nodes():
index = Index(INDEX_BBOX, max_items=1)
index.insert(ITEM1, BBOX1)
index.insert(ITEM2, INDEX_BBOX)
index.remove(ITEM1, BBOX1)
index.remove(ITEM2, INDEX_BBOX)
assert len(index) == 0
assert index.intersect(INDEX_BBOX) == []
def test_should_add_multiple_nodes_and_find_them():
index = Index(INDEX_BBOX)
index.insert(ITEM1, BBOX1)
index.insert(ITEM2, BBOX2)
assert len(index) == 2
assert index.intersect(BBOX1) == {ITEM1}
assert index.intersect(BBOX2) == {ITEM2}
assert index.intersect(INDEX_BBOX) == {ITEM1, ITEM2}
def combine_osm_components(road,radius = 0.01):
"""
Combines network components by finding nearest nodes
based on a QD Tree Approach.
Parameters
----------
graph : TYPE
DESCRIPTION.
Returns
-------
None.
"""
# Initialize QD Tree
longitudes = [road.nodes[n]['x'] for n in road.nodes()]
latitudes = [road.nodes[n]['y'] for n in road.nodes()]
xmin = min(longitudes); xmax = max(longitudes)
ymin = min(latitudes); ymax = max(latitudes)
bbox = (xmin,ymin,xmax,ymax)
idx = Index(bbox)
# Differentiate large and small components
comps = [c for c in list(nx.connected_components(road))]
lencomps = [len(c) for c in list(nx.connected_components(road))]
indlarge = lencomps.index(max(lencomps))
node_main = list(road.subgraph(comps[indlarge]).nodes())
del(comps[indlarge])
# keep track of nodes so we can recover them later
nodes = []
# create bounding box around each point in large component
for i,node in enumerate(node_main):
pt = Point([road.nodes[node]['x'],road.nodes[node]['y']])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
idx.insert(i, pt_bounds)
nodes.append((pt, pt_bounds, node))
# find intersection and add edges
edgelist = []
for c in comps:
node_comp = list(road.subgraph(c).nodes())
nodepairs = []
for n in node_comp:
pt = Point([road.nodes[n]['x'],road.nodes[n]['y']])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
matches = idx.intersect(pt_bounds)
closest_pt = min(matches,key=lambda i: nodes[i][0].distance(pt))
nodepairs.append((n,nodes[closest_pt][-1]))
# Get the geodesic distance
dist = [MeasureDistance([road.nodes[p[0]]['x'],road.nodes[p[0]]['y']],
[road.nodes[p[1]]['x'],road.nodes[p[1]]['y']]) \
for p in nodepairs]
edgelist.append(nodepairs[np.argmin(dist)]+tuple([0]))
return edgelist
def test_should_empty_index_after_removing_multiple_added_nodes_in_multiple_horizontal_quad_nodes():
index = Index(INDEX_BBOX, max_items=1)
bbox2 = (0, 0, INDEX_BBOX[2], 1)
index.insert(ITEM1, BBOX1)
index.insert(ITEM2, bbox2)
index.remove(ITEM1, BBOX1)
index.remove(ITEM2, bbox2)
assert len(index) == 0
assert index.intersect(INDEX_BBOX) == []
def test_should_add_multiple_nodes_and_find_them():
index = Index(INDEX_BBOX)
index.insert(ITEM1, BBOX1)
index.insert(ITEM2, BBOX2)
assert len(index) == 2
assert index.intersect(BBOX1) == [ITEM1]
assert index.intersect(BBOX2) == [ITEM2]
res = index.intersect(INDEX_BBOX)
assert ITEM1 in res
assert ITEM2 in res
def remove_overlapping_points(missed_points_coord):
missed_points_qtree = Index(bbox=(np.amin(missed_points_coord[:, 0]), np.amin(missed_points_coord[:,1]), np.amax(missed_points_coord[:,0]), np.amax(missed_points_coord[:, 1])))
d_y = 1/444444.0 ## 0.25 meters
d_x = np.cos(missed_points_coord[0][1] / (180 / np.pi))/444444.0 ## 0.25 meters
for i in trange(len(missed_points_coord), desc='check for double points'):
mp = missed_points_coord[i]
if not missed_points_qtree.intersect((mp[0] - d_x, mp[1] - d_y, mp[0] + d_x, mp[1] + d_y)):
missed_points_qtree.insert(mp, (mp[0] - d_x, mp[1] - d_y, mp[0] + d_x, mp[1] + d_y))
missed_points_coord = missed_points_qtree.intersect(bbox=(np.amin(missed_points_coord[:, 0]), np.amin(missed_points_coord[:,1]), np.amax(missed_points_coord[:,0]), np.amax(missed_points_coord[:, 1])))
return missed_points_coord
def combine_components(graph,cords,radius = 0.01):
"""
Combines network components by finding nearest nodes
based on a QD Tree Approach.
Parameters
----------
graph : TYPE
DESCRIPTION.
Returns
-------
None.
"""
# Initialize QD Tree
xmin,ymin = np.min(np.array(list(cords.values())),axis=0)
xmax,ymax = np.max(np.array(list(cords.values())),axis=0)
bbox = (xmin,ymin,xmax,ymax)
idx = Index(bbox)
# Differentiate large and small components
comps = [c for c in list(nx.connected_components(graph))]
lencomps = [len(c) for c in list(nx.connected_components(graph))]
indlarge = lencomps.index(max(lencomps))
node_main = list(graph.subgraph(comps[indlarge]).nodes())
del(comps[indlarge])
# keep track of lines so we can recover them later
nodes = []
# create bounding box around each point in large component
for i,node in enumerate(node_main):
pt = Point(cords[node])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
idx.insert(i, pt_bounds)
nodes.append((pt, pt_bounds, node))
# find intersection and add edges
edgelist = []
for c in comps:
node_comp = list(graph.subgraph(c).nodes())
nodepairs = []
for n in node_comp:
pt = Point(cords[n])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
matches = idx.intersect(pt_bounds)
closest_pt = min(matches,key=lambda i: nodes[i][0].distance(pt))
nodepairs.append((n,nodes[closest_pt][-1]))
dist = [MeasureDistance(cords[p[0]],cords[p[1]]) for p in nodepairs]
edgelist.append(nodepairs[np.argmin(dist)])
return edgelist
def build_qtree(self):
bounds_prep = prep(self.bounds)
bbox = []
for val in self.bounds.bounds:
bbox.append(val * 2)
#ret = Index(bbox=bbox, maxdepth=100000)
ret = Index(bbox=bbox)
for segment in self.parentage.nodes():
if segment.abs_polygon:
seg_poly = segment.abs_polygon
if not bounds_prep.disjoint(seg_poly):
ret.insert(segment, segment.abs_polygon.bounds)
return ret
def update(self):
collisionIndex = Index((0, 0, 800, 600))
for o in self.gameObjects:
o.update()
if o.collidable:
collisionIndex.insert(
o, (o.rect.left, o.rect.top, o.rect.right, o.rect.bottom))
for i in self.collidableObjects:
collisions = collisionIndex.intersect(
(i.rect.left, i.rect.top, i.rect.right, i.rect.bottom))
if (len(collisions) > 1): # Intersecting more than self
# logging.info("Collision!")
CollisionHandler(collisions)
def create_quadtree(file):
# go through the file and create a dot object for each line and add it to a list
dot_list = []
for line in file:
line = line.split()
x = float(line[0])
y = float(line[1])
label = line[2]
dot_list.append(Dot(label, (x, y, x, y)))
file.close()
# add all dots to the quadtree
spindex = Index(bbox=(0, 0, 1, 1))
for dot in dot_list:
spindex.insert(dot, dot.bbox)
return spindex
def remove_overlapping_points(new_points, missed_points_coord):
d_y = 1/222222.0 ## 0.5 meters
d_x = np.cos(missed_points_coord[0][1] / (180 / np.pi))/222222.0 ## 0.5 meters
missed_new_points_qtree = Index(bbox=(np.amin(new_points[:,0]), np.amin(new_points[:,1]), np.amax(new_points[:,0]), np.amax(new_points[:,1])))
for n in new_points:
if not missed_new_points_qtree.intersect((n[0] - d_x, n[1] - d_y, n[0] + d_x, n[1] + d_y)):
missed_new_points_qtree.insert(n, bbox=(n[0], n[1], n[0], n[1]))
new_points = missed_new_points_qtree.intersect((np.amin(new_points[:,0]), np.amin(new_points[:,1]), np.amax(new_points[:,0]), np.amax(new_points[:,1])))
missed_points_qtree = Index(bbox=(np.amin(missed_points_coord[:, 0]), np.amin(missed_points_coord[:,1]), np.amax(missed_points_coord[:,0]), np.amax(missed_points_coord[:, 1])))
checked_points = []
for p in missed_points_coord:
missed_points_qtree.insert(p, (p[0], p[1], p[0], p[1]))
for i in trange(len(new_points), desc='check for double points'):
mp = new_points[i]
if not missed_points_qtree.intersect((mp[0] - d_x, mp[1] - d_y, mp[0] + d_x, mp[1] + d_y)):
checked_points.append(mp)
return checked_points
def update(self):
qt = Index(bbox=(0, 0, self.w, self.h), max_items=5)
for i in range(len(self.people)):
s = self.people[i]
qt.insert(i, (s.x, s.y, s.x, s.y))
s.update(self.peopleSpeed)
for s in self.people:
if self.distanceRadius > 0: # 사회적 거리두기
distance = self.distanceRadius
for i in qt.intersect((s.x - distance, s.y - distance,
s.x + distance, s.y + distance)):
other = self.people[i]
if np.sqrt((s.x - other.x) * (s.x - other.x) +
(s.y - other.y) * (s.y - other.y)) > distance:
continue
if s != self.people[i]:
direct = np.arctan2(s.y - self.people[i].y,
s.x - self.people[i].x)
s.dx += np.cos(direct) * 2
s.dy += np.sin(direct) * 2
mag = np.sqrt(s.dx * s.dx + s.dy * s.dy)
s.dx /= mag
s.dy /= mag
for i in qt.intersect(
(s.x - self.infectionRadius, s.y - self.infectionRadius,
s.x + self.infectionRadius, s.y + self.infectionRadius)):
other = self.people[i]
if np.sqrt((s.x - other.x) * (s.x - other.x) +
(s.y - other.y) *
(s.y - other.y)) > self.infectionRadius:
continue
if s.status == 1 and np.random.rand() < self.infectionPercent:
self.people[i].GetInfection(self.recoverTime,
self.reinfectionPercent)
self.setLogData()
def __apply_quadtree_pyqtree(self, ntwk, ltype):
ntwk_w_data_ltype = ntwk[ntwk['LinkID_ptv'].isin(
self._links_with_data)]
print 'ltype', ltype, '# of links w/ data =', ntwk_w_data_ltype.shape[
0]
spindex = Index(bbox=self._bbox,
max_items=int(
min(ntwk_w_data_ltype.shape[0] * PCT_LINKS_IN_CELL,
MAX_CROWD)),
max_depth=MAX_DEPTH)
for i in xrange(ntwk_w_data_ltype.shape[0]):
#print ltype, i, '/ ', ntwk_w_data_ltype.shape[0], '(', float(i)/ntwk_w_data_ltype.shape[0]*100, '%)'
link = ntwk_w_data_ltype.iloc[i]
spindex.insert(link['LinkID_ptv'], bbox=(link['centroid'] * 2))
link_cellid_map = self.__get_link_cube_tuple(spindex, ltype)
assert len(link_cellid_map) == ntwk_w_data_ltype.shape[0]
#now assign cell id;s to the links without data by looking at their nearest neighboring link's cell_id
link_cellid_map = self.__find_cellid_neighbors(spindex, ntwk,
link_cellid_map, ltype)
return link_cellid_map
class BlockSearch(object):
def __init__(self, blocks):
bboxs = [block.bounding_box for block in blocks]
xmax = max([bb.x + bb.width for bb in bboxs])
ymax = max([bb.y + bb.height for bb in bboxs])
self.spindex = PqtreeIndex(bbox=(0, 0, xmax, ymax))
self.wrapper_map = {}
for block in blocks:
wrapper = DeletableWrapper(block)
self.wrapper_map[block] = wrapper
self.spindex.insert(wrapper, _to_bbox(block.bounding_box))
def find_intersection_with(self, search_bounding_box):
return [
wrapper.data for wrapper in self.spindex.intersect(
_to_bbox(search_bounding_box)) if not wrapper.deleted
]
def remove(self, block):
wrapper = self.wrapper_map.get(block)
if wrapper is not None:
wrapper.deleted = True
class TreeStruct:
bbox = (-10000, -10000, 10000, 10000)
##
# Maintain link from points to rectangle objects?
def __init__(self):
self.index = Index(bbox=self.bbox, max_items=40, max_depth=15)
def addRect(self, rect):
if (bboxoutside(rect.bbox, self.bbox)):
print('outside')
pass # TODO
self.index.insert(rect, rect.bbox)
def removeRect(self, rect):
self.index.remove(rect, rect.bbox)
def query(self, rect):
candidates = self.index.intersect(rect.bbox)
# return candidates
# TBD if we want to make the check both ways or are okay with overlaps only being detected on one end
return [candidate for candidate in candidates if rect.overlaps(candidate) or candidate.overlaps(rect)]
def create_quadTree(chr):
print("processing ", chr)
chromLength = chromosomes[chr]
dims = 2
hlevel = math.ceil(math.log2(chromLength) / dims)
print("hlevel", hlevel)
x_y_dim = math.ceil(math.pow(2, hlevel))
print("max x|y =", x_y_dim)
# f = open("quadtree/chrmids.json")
# chromIndexes = json.loads(f)
# tree = Index(bbox=(0, 0, x_y_dim, x_y_dim), disk="quadtree/indexes/roadmap." + chr + ".quadtree.index", first_run=True)
tree = Index(bbox=(0, 0, x_y_dim, x_y_dim),
disk="./roadmap." + chr + ".quadtree.index")
for file in chromIndexes.keys():
print("\t file - ", file)
if chr in chromIndexes[file]:
chrmId = chromIndexes[file][chr]
df = pandas.read_csv("quadtree/processed/" + file + ".leaves",
header=0)
df = df[df["rStartChromIx"] == chrmId]
print("\t df shape - ", df.shape)
for i, row in df.iterrows():
# print(row)
x_start, y_start, _ = hcoords(row["rStartBase"], chromLength)
x_end, y_end, hlevel = hcoords(row["rEndBase"], chromLength)
bbox = range2bbox(hlevel, {
"start": row["rStartBase"],
"end": row["rEndBase"]
})
tree.insert(
(row["rStartBase"], row["rEndBase"], row["rdataOffset"],
row["rDataSize"], fileIds[file]), bbox)
else:
print("\t !!!!!!! chrm doesn't exist - ", file)
def test_should_empty_index_after_removing_added_node():
index = Index(INDEX_BBOX)
index.insert(ITEM1, BBOX1)
index.remove(ITEM1, BBOX1)
assert len(index) == 0
assert index.intersect(BBOX1) == []
multiple=True)
plants_path = filedialog.askopenfilename(
initialdir=r"Z:\800 Operational\c07_hollandbean\Joke visser",
title="Select plant file",
parent=root)
root.destroy()
plants = geopandas.read_file(plants_path)
plants = plants.to_crs({'init': 'epsg:28992'})
spindex = Index(
Polygon([(p.x, p.y) for p in plants.loc[:, 'geometry']]).bounds)
for i in plants.index:
spindex.insert(
{
'index': i,
'plant': list(plants.loc[i, 'geometry'].coords)[0]
}, (list(plants.loc[i, 'geometry'].coords)[0][0],
list(plants.loc[i, 'geometry'].coords)[0][1],
list(plants.loc[i, 'geometry'].coords)[0][0],
list(plants.loc[i, 'geometry'].coords)[0][1]))
pbar = tqdm(total=len(polys_paths), desc="Computing areas", position=0)
for path in polys_paths:
polys = geopandas.read_file(path)
polys = polys.to_crs({'init': 'epsg:28992'})
key = 'area' + re.findall(r'\d+', path)[-1]
pbar2 = tqdm(total=len(polys.index), desc="computing areas", position=0)
for poly in polys.loc[:, 'geometry']:
if poly.is_valid:
plants_i = spindex.intersect(poly.bounds)
if isinstance(plants_i, list):
n = sum([
hlevel = math.ceil(math.log2(chromLength) / dims)
print("hlevel", hlevel)
x_y_dim = math.ceil(math.pow(2, hlevel))
print("max x|y =", x_y_dim)
tree = Index(bbox=(0, 0, x_y_dim, x_y_dim))
data = pickle.load(open("result1.p", "rb"))
print(len(data))
for entry in data:
print(entry)
(start, end, offset, length, fileName) = (entry[0], entry[1], entry[2],
entry[3], 1)
x, y, _ = hcoords(start, chromLength)
print("x,y", x, y)
tree.insert((start, end, offset, length, fileName), (x, y, x + 1, y + 1))
data = pickle.load(open("result2.p", "rb"))
print(len(data))
for entry in data:
print(entry)
(start, end, offset, length, fileName) = (entry[1], entry[3], entry[4],
entry[5], 2)
x, y, _ = hcoords(start, chromLength)
print("x,y", x, y)
tree.insert((start, end, offset, length, fileName), (x, y, x + 1, y + 1))
# repeating to see overhead
data = pickle.load(open("result1.p", "rb"))
print(len(data))
for entry in data:
# index_c += [key for _ in range(len(contained_vor_polys))]
# vor_polys = list(filter(lambda p : not prepped.contains(p), vor_polys))
# intersecting_vor_polys = list(filter(lambda p : prepped.intersects(p), vor_polys))
# ordered_intersecting_vor_polys += intersecting_vor_polys
# index_i += [key for _ in range(len(intersecting_vor_polys))]
# vor_polys = list(filter(lambda p: not prepped.intersects(p), vor_polys))
# ordered[key]['vor_polys_i'] = intersecting_vor_polys
# pbar2.update(len(contained_vor_polys)+len(intersecting_vor_polys))
# pbar2.close()
# =============================================================================
points_list = [[] for _ in range(len(polys))]
spindex = Index(
Polygon(utilv.readable_values_inv(vor.points, mx, my)).bounds)
for i, p in enumerate(utilv.readable_values_inv(vor.points, mx, my)):
spindex.insert({'index': i, 'coord': p}, (p[0], p[1], p[0], p[1]))
pbar15 = tqdm(total=len(ordered), desc="sorting", position=0)
ints = []
for i, key in enumerate(ordered.keys()):
if (isinstance(ordered[key]['boundary'], Polygon)
and ordered[key]['boundary'].exterior) or isinstance(
ordered[key]['boundary'], MultiPolygon):
prepped = prep(ordered[key]['boundary'])
filtered = list(
filter(
lambda a: prepped.contains(
Point(a['coord'][0], a['coord'][1])),
spindex.intersect(ordered[key]['boundary'].bounds)))
points_list[i].extend([p['index'] for p in filtered])
pbar15.update(1)
class Tiler:
def set_extent(self, tags):
self.max_x = max((tag.x for tag in tags)) + SHIFT
self.min_x = min((tag.x for tag in tags)) - SHIFT
self.max_y = max((tag.y for tag in tags)) + SHIFT
self.min_y = min((tag.y for tag in tags)) - SHIFT
self.origin = Point(self.min_x, self.min_y)
max_size = max(self.max_x - self.min_x, self.max_y - self.min_y)
self.map_size = max_size
self.tile_size = [self.map_size / (1 << i) * METATILE_SIZE for i in range(20)]
self.tag_to_normpos = dict()
for tag in tags:
x, y = tag.x, tag.y
rel_x, rel_y = x - self.origin.x, y - self.origin.y
norm_x, norm_y = rel_x / self.map_size, rel_y / self.map_size
self.tag_to_normpos[tag.name] = (norm_x, norm_y)
def set_bbox(self, tags):
self.tag_spatial_index = Index(bbox=(self.min_x, self.min_y, self.max_x, self.max_y))
for tag in tags:
bbox = (tag.x, tag.y, tag.x, tag.y)
self.tag_spatial_index.insert(tag, bbox)
def set_postcount(self, tags):
for tag in tags:
tag.PostCount = int(getattr(tag, 'PostCount', -1))
self.max_post_count = max(int(tag.PostCount) for tag in tags)
def set_fonts(self):
path_to_font = os.path.join(os.path.dirname(os.path.abspath(__file__)), './Verdana.ttf')
self.fonts = [ImageFont.truetype(path_to_font, ANTIALIASING_SCALE * (25 - zoom * 2)) for zoom in range(8 + 1)]
def __init__(self, tags):
self.set_extent(tags)
self.set_bbox(tags)
self.set_postcount(tags)
self.set_fonts()
def search(self, name):
return self.tag_to_normpos.get(name, '')
def get_postcount_measure(self, tag):
tag_count = tag.PostCount
max_post_count = self.max_post_count
return tag_count / max_post_count
def get_tags_in_tile(self, meta_x, meta_y, zoom, with_shift):
tile_size = self.tile_size[zoom]
lower_left_corner = Point(self.origin.x + meta_x * tile_size,
self.origin.y + meta_y * tile_size)
shift = SHIFT if with_shift else 0
tags_inside_tile = self.tag_spatial_index.intersect((lower_left_corner.x - shift,
lower_left_corner.y - shift,
lower_left_corner.x + tile_size + shift,
lower_left_corner.y + tile_size + shift))
return tags_inside_tile
def get_names_of_shown_tags(self, meta_x, meta_y, zoom):
"""
Return the names of tags that we will show on the map.
On low zoom levels, not all names are shown.
"""
if zoom >= ZOOM_TEXT_SHOW:
# We know that all tag names will be shown anyway, so just return.
return []
tags_inside_tile = self.get_tags_in_tile(meta_x, meta_y, zoom, False)
all_postcounts = sorted([(tag.PostCount, tag.name) for tag in tags_inside_tile])
largest_tags = heapq.nlargest(TAGS_ANNOTATED_PER_TILE, all_postcounts)
return {x[1] for x in largest_tags if x[0] > 0}
def get_metatile(self, meta_x, meta_y, zoom):
'''
Get 8x8 rectangle of tiles, compute them at once.
This is faster than computing them one-by-one.
`meta_x` and `meta_y` are coordinates of upper-left tile of the
generated metatile.
'''
meta_x /= METATILE_SIZE
meta_y /= METATILE_SIZE
img = Image.new('RGB', (TILE_DIM * METATILE_SIZE,
TILE_DIM * METATILE_SIZE), (240, 240, 240))
draw = ImageDraw.Draw(img)
tile_size = self.tile_size[zoom]
lower_left_corner = Point(self.origin.x + meta_x * tile_size,
self.origin.y + meta_y * tile_size)
max_circle_rad = zoom * 1
cnt_points = 0
names_of_shown_tags = set()
for dx in (-1, 0, 1):
for dy in (-1, 0, 1):
names_of_shown_tags.update(self.get_names_of_shown_tags(meta_x + dx, meta_y + dy, zoom))
# Match slightly more tags, so that circles from neighbouring tiles can be drawn partially.
tags_inside_tile = self.get_tags_in_tile(meta_x, meta_y, zoom, True)
def get_point_from_tag(tag):
x, y = tag.x, tag.y
# Get coordinates from tile origin.
point_coords = Point(x - lower_left_corner.x, y - lower_left_corner.y)
# Now scale to TILE_DIM.
pnt = Point(point_coords.x / tile_size * TILE_DIM * METATILE_SIZE,
point_coords.y / tile_size * TILE_DIM * METATILE_SIZE)
return pnt
for tag in tags_inside_tile:
pnt = get_point_from_tag(tag)
# Heuristic formula for showing post counts by circle sizes.
post_count_measure = self.get_postcount_measure(tag)
circle_rad = max(0.5, max_circle_rad * post_count_measure)
draw.ellipse([pnt.x - circle_rad, pnt.y - circle_rad,
pnt.x + circle_rad, pnt.y + circle_rad],
fill=(122, 176, 42))
cnt_points += 1
# Draw text after all circles, so that it is not overwritten.
# (because I did not find any kind of z-index feature in PIL)
for tag in tags_inside_tile:
pnt = get_point_from_tag(tag)
fill = (0, 0, 0)
if zoom >= ZOOM_TEXT_SHOW or tag.name in names_of_shown_tags:
draw.text(pnt, tag.name, fill=fill, font=self.fonts[zoom])
del draw
return img, cnt_points
class MapLink:
"""
This class consists of attributes and methods to evaluate the nearest road
network link to a point. The point may be a home location or a substation.
The algorithm uses a QD-Tree approach to evaluate a bounding box for each
point and link then finds the nearest link to each point.
"""
def __init__(self,road,radius=0.01):
'''
'''
longitudes = [road.nodes[n]['x'] for n in road.nodes()]
latitudes = [road.nodes[n]['y'] for n in road.nodes()]
xmin = min(longitudes); xmax = max(longitudes)
ymin = min(latitudes); ymax = max(latitudes)
bbox = (xmin,ymin,xmax,ymax)
# keep track of lines so we can recover them later
all_link = list(road.edges())
self.lines = []
# initialize the quadtree index
self.idx = Index(bbox)
# add edge bounding boxes to the index
for i, link in enumerate(all_link):
# create line geometry
link_geom = road.edges[link]['geometry']
# bounding boxes, with padding
x1, y1, x2, y2 = link_geom.bounds
bounds = x1-radius, y1-radius, x2+radius, y2+radius
# add to quadtree
self.idx.insert(i, bounds)
# save the line for later use
self.lines.append((link_geom, bounds, link))
return
def map_point(self,points,path=os.getcwd(),fiscode='121',
radius=0.01):
'''
Finds the nearest link to the point under consideration and saves the
map as a csv file in the specified location.
'''
Map2Link = {}
for h in points.cord:
pt = Point(points.cord[h])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
matches = self.idx.intersect(pt_bounds)
# find closest path
try:
closest_path = min(matches,
key=lambda i: self.lines[i][0].distance(pt))
Map2Link[h] = self.lines[closest_path][-1]
except:
Map2Link[h] = None
# Delete unmapped points
unmapped = [p for p in Map2Link if Map2Link[p]==None]
for p in unmapped:
del Map2Link[p]
# Save as a csv file
df_map = pd.DataFrame.from_dict(Map2Link,orient='index',
columns=['source','target'])
df_map.index.names = ['hid']
df_map.to_csv(path+fiscode+'-home2link.csv')
return Map2Link
class MapOSM:
"""
Class consisting of attributes and methods to map OSM links to the residences
"""
def __init__(self,road,radius=0.01):
"""
Initializes the class object by creating a bounding box of known radius
around each OSM road link.
Parameters
----------
road : networkx Multigraph
The Open Street Map multigraph with node and edge attributes.
radius : float, optional
The radius of the bounding box around each road link.
The default is 0.01.
Returns
-------
None.
"""
longitudes = [road.nodes[n]['x'] for n in road.nodes()]
latitudes = [road.nodes[n]['y'] for n in road.nodes()]
xmin = min(longitudes); xmax = max(longitudes)
ymin = min(latitudes); ymax = max(latitudes)
bbox = (xmin,ymin,xmax,ymax)
# keep track of edges so we can recover them later
all_link = list(road.edges(keys=True))
self.links = []
# initialize the quadtree index
self.idx = Index(bbox)
# add edge bounding boxes to the index
for i, link in enumerate(all_link):
# create line geometry
link_geom = road.edges[link]['geometry']
# bounding boxes, with padding
x1, y1, x2, y2 = link_geom.bounds
bounds = x1-radius, y1-radius, x2+radius, y2+radius
# add to quadtree
self.idx.insert(i, bounds)
# save the line for later use
self.links.append((link_geom, bounds, link))
return
def map_point(self,points,path=os.getcwd(),fiscode='121',
radius=0.01):
'''
Finds the nearest link to the residence under consideration and saves the
map as a csv file in the specified location.
'''
Map2Link = {}
for h in points.cord:
pt = Point(points.cord[h])
pt_bounds = pt.x-radius, pt.y-radius, pt.x+radius, pt.y+radius
matches = self.idx.intersect(pt_bounds)
# find closest path
try:
closest_path = min(matches,
key=lambda i: self.links[i][0].distance(pt))
Map2Link[h] = self.links[closest_path][-1]
except:
Map2Link[h] = None
# Delete unmapped points
unmapped = [p for p in Map2Link if Map2Link[p]==None]
for p in unmapped:
del Map2Link[p]
# Save as a csv file
df_map = pd.DataFrame.from_dict(Map2Link,orient='index',
columns=['source','target','key'])
df_map.index.names = ['hid']
df_map.to_csv(path+fiscode+'-home2OSM.csv')
return Map2Link
import csv
from pyqtree import Index
from pcpoint import PostCodePoint
pcFile = "postcodes.csv"
# Latitudes: 54.596553 to 54.603728
# Longitudes: -5.937032 to -5.922495
pcBounds = (54.596552, -5.937033, 54.603729, -5.922494)
pcIndex = Index(bbox = pcBounds)
with open(pcFile) as csvfile:
pcReader = csv.DictReader(csvfile)
print "populating qtree"
for row in pcReader:
postcode = PostCodePoint(row['Postcode'])
lat = row['Latitude']
lon = row['Longitude']
pcIndex.insert(postcode, (lat, lon, lat, lon))
print "qtree populated"
print "attempt delaunay?"
rural_blocks = data_blocks.loc[data_blocks.UR10 == 'R']["geometry"].values
urban_blocks = data_blocks.loc[data_blocks.UR10 == 'U']["geometry"].values
#%% Using QDTree
subx = [subs.iloc[i]["geometry"].coords[0][0] for i in range(len(subs))]
suby = [subs.iloc[i]["geometry"].coords[0][1] for i in range(len(subs))]
xmax = max(subx)
xmin = min(subx)
ymax = max(suby)
ymin = min(suby)
bbox = (xmin, ymin, xmax, ymax)
# Urban substations
idx = Index(bbox)
for pos, poly in enumerate(urban_blocks):
idx.insert(pos, poly.bounds)
#iterate through points
urban_subs = []
for i in range(len(subs)):
point = subs.iloc[i]["geometry"]
# iterate through spatial index
for j in idx.intersect(point.coords[0]):
if point.within(urban_blocks[j]):
urban_subs.append(subs.iloc[i]["ID"])
# Rural substations
idx = Index(bbox)
for pos, poly in enumerate(rural_blocks):
idx.insert(pos, poly.bounds)
df['area' + re.findall(r'\d+', p)[-1]] = None
keys.append('area' + re.findall(r'\d+', p)[-1])
f = gpd.read_file(p)
f = f.to_crs({'init': 'epsg:28992'})
# f = gpd.GeoDataFrame(f)
f['isvalid'] = f.geometry.apply(lambda x: x.is_valid)
f = f[(f['isvalid'] == True)]
f = f.drop(columns='isvalid')
polys_list.append(f.loc[:, 'geometry'])
areas.append([0 for _ in range(len(plants))])
pbar0.update(1)
pbar0.close()
pyq = Index(bbox=bbox)
for i, p in enumerate(plants):
pyq.insert(i, p.bounds)
convex_hull = Polygon([(p.x, p.y) for p in plants]).convex_hull
pbar = tqdm(total=len(voronoi_paths), position=0)
for j, f in enumerate(polys_list):
pbar1 = tqdm(total=len(polys_list[j]), position=0)
for p in f:
if (p.is_valid and p.within(convex_hull)) or (
not p.is_valid and p.buffer(0).within(convex_hull)):
bbox = p.bounds
intersected = pyq.intersect(bbox)
if intersected:
for k in intersected:
areas[j][k] = p.area / len(intersected)
pbar1.update(1)
self.fileName = fileName
def __repr__(self):
return str((self.start, self.end, self.offset, self.fileName))
tree = Index(bbox=(0, 0, 16001, 16001))
data = pickle.load(open("result1.p", "rb"))
print(len(data))
for entry in data:
# print(entry)
(start, end, offset, length, fileName) = (entry[0], entry[1], entry[2],
entry[3], 1)
tree.insert(
(start, end, offset, length, fileName),
(start // 16000, start % 16000, start // 16000 + 1, start % 16000 + 1))
# data = pickle.load(open( "result2.p", "rb"))
# print(len(data))
# for entry in data:
# # print(entry)
# (start, end, offset, length, fileName) = (entry[1], entry[3], entry[4], entry[5], "39031")
# tree.insert((start, end, offset, length, fileName), (start//16000, start%16000, start//16000 + 1, start%16000 + 1))
overlapbbox = (1, 1, 860, 860)
matches = tree.intersect(overlapbbox)
print(matches[0])
for item in matches:
print(sys.getsizeof(item))
def test_should_add_single_node_and_find_its_intersection():
index = Index(INDEX_BBOX)
index.insert(ITEM1, BBOX1)
assert len(index) == 1
assert index.intersect(BBOX1) == [ITEM1]