def generate_children(self):
"""Generates the fours child files for this child if they don't already exist."""
if self.children is not None:
return
print "Generating children for %s (%s rows)" % (self.bbox, self.count)
self.children = [QuadtreeNode(self.tree, b)
for b in self.bounds.get_children()]
with utils.msgpack_open(self.source_filename) as f:
with utils.msgpack_open(self.children[0].source_filename, "w") as self.children[0].file:
with utils.msgpack_open(self.children[1].source_filename, "w") as self.children[1].file:
with utils.msgpack_open(self.children[2].source_filename, "w") as self.children[2].file:
with utils.msgpack_open(self.children[3].source_filename, "w") as self.children[3].file:
for row in f:
for child in self.children:
if self.tree.latitude_col in row and self.tree.longitude_col in row and child.bbox.contains(row[self.tree.longitude_col], row[self.tree.latitude_col]):
child.file.write(row)
child.count += 1
break
for child in self.children:
del child.file
return self.children
def load(cls):
with utils.msgpack_open("tree.msg") as f:
spec = f.next()
self = cls(spec.pop("filename"), __bare__ = True, **spec)
self.root = quad_tree_node.QuadtreeNode(self)
self.root.load()
return self
def generate_tile_from_child_tiles(self):
clusters = {}
for child in self.children:
with utils.msgpack_open(child.cluster_filename) as f:
for row in f:
cluster = cluster_mod.Cluster.from_cluster_row(row)
row = cluster.get_row()
gridcode = str(vectortile.TileBounds.from_point(row[self.tree.longitude_col], row[self.tree.latitude_col], self.bounds.zoom_level + self.tree.clustering_levels))
if gridcode not in clusters:
clusters[gridcode] = cluster
else:
clusters[gridcode] += cluster
# Merge clusters until we have few enough for a tile
while len(clusters) > self.tree.max_count:
newclusters = {}
for gridcode, cluster in clusters.iteritems():
gridcode = gridcode[:-1]
if gridcode not in newclusters:
newclusters[gridcode] = cluster
else:
newclusters[gridcode] += cluster
clusters = newclusters
self.write_tile(clusters.values())
def save(self):
with utils.msgpack_open("tree.msg", "w") as f:
f.write({"max_depth": self.max_depth,
"max_count": self.max_count,
"remove": self.remove,
"clustering_levels": self.clustering_levels,
"filename": self.filename,
})
self.root.save()
def save(self):
with utils.msgpack_open(self.info_filename, "w") as f:
f.write({"bounds": str(self.bounds),
"count": self.count,
"colsByName": self.colsByName
})
if self.children is not None:
for child in self.children:
child.save()
def write_tile(self, clusters):
with utils.msgpack_open(self.cluster_filename, "w") as f:
for cluster in clusters:
f.write(cluster.get_cluster_row())
with open(self.tile_filename, "w") as f:
data = [self.update_colsByName(self.tree.map_row(cluster.get_row()))
for cluster in clusters]
f.write(str(
vectortile.Tile.fromdata(
data,
{"colsByName": self.colsByName})))
def load(self):
with utils.msgpack_open(self.info_filename) as f:
info = f.next()
self.count = info['count']
self.colsByName = info['colsByName']
self.children = []
for child_bounds in self.bounds.get_children():
child = QuadtreeNode(self.tree, child_bounds)
if os.path.exists(child.info_filename):
self.children.append(child)
child.load()
if not self.children:
self.children = None
def __init__(self, filename, __bare__ = False, **kw):
self.filename = filename
for key, value in kw.iteritems():
setattr(self, key, value)
if __bare__: return
self.root = quad_tree_node.QuadtreeNode(self)
print "Loading data..."
with utils.msgpack_open(self.root.source_filename, "w") as outf:
with gpsdio.open(filename) as f:
for row in f:
out_row = {}
for key, value in row.iteritems():
if isinstance(value, datetime.datetime):
value = float(value.strftime("%s")) * 1000.0
if not isinstance(value, (float, int, bool)):
continue
out_row[key] = value
outf.write(out_row)
self.root.count += 1
def generate_tile_from_source(self):
with utils.msgpack_open(self.source_filename) as f:
rows = list(f)
self.write_tile([cluster_mod.Cluster.from_row(row) for row in rows])
