def generate_children(self): """Generates the fours child files for this child if they don't already exist.""" if self.children is not None: return print "Generating children for %s (%s rows)" % (self.bbox, self.count) self.children = [QuadtreeNode(self.tree, b) for b in self.bounds.get_children()] with utils.msgpack_open(self.source_filename) as f: with utils.msgpack_open(self.children[0].source_filename, "w") as self.children[0].file: with utils.msgpack_open(self.children[1].source_filename, "w") as self.children[1].file: with utils.msgpack_open(self.children[2].source_filename, "w") as self.children[2].file: with utils.msgpack_open(self.children[3].source_filename, "w") as self.children[3].file: for row in f: for child in self.children: if self.tree.latitude_col in row and self.tree.longitude_col in row and child.bbox.contains(row[self.tree.longitude_col], row[self.tree.latitude_col]): child.file.write(row) child.count += 1 break for child in self.children: del child.file return self.children
def load(cls): with utils.msgpack_open("tree.msg") as f: spec = f.next() self = cls(spec.pop("filename"), __bare__ = True, **spec) self.root = quad_tree_node.QuadtreeNode(self) self.root.load() return self
def generate_tile_from_child_tiles(self): clusters = {} for child in self.children: with utils.msgpack_open(child.cluster_filename) as f: for row in f: cluster = cluster_mod.Cluster.from_cluster_row(row) row = cluster.get_row() gridcode = str(vectortile.TileBounds.from_point(row[self.tree.longitude_col], row[self.tree.latitude_col], self.bounds.zoom_level + self.tree.clustering_levels)) if gridcode not in clusters: clusters[gridcode] = cluster else: clusters[gridcode] += cluster # Merge clusters until we have few enough for a tile while len(clusters) > self.tree.max_count: newclusters = {} for gridcode, cluster in clusters.iteritems(): gridcode = gridcode[:-1] if gridcode not in newclusters: newclusters[gridcode] = cluster else: newclusters[gridcode] += cluster clusters = newclusters self.write_tile(clusters.values())
def save(self): with utils.msgpack_open("tree.msg", "w") as f: f.write({"max_depth": self.max_depth, "max_count": self.max_count, "remove": self.remove, "clustering_levels": self.clustering_levels, "filename": self.filename, }) self.root.save()
def save(self): with utils.msgpack_open(self.info_filename, "w") as f: f.write({"bounds": str(self.bounds), "count": self.count, "colsByName": self.colsByName }) if self.children is not None: for child in self.children: child.save()
def write_tile(self, clusters): with utils.msgpack_open(self.cluster_filename, "w") as f: for cluster in clusters: f.write(cluster.get_cluster_row()) with open(self.tile_filename, "w") as f: data = [self.update_colsByName(self.tree.map_row(cluster.get_row())) for cluster in clusters] f.write(str( vectortile.Tile.fromdata( data, {"colsByName": self.colsByName})))
def load(self): with utils.msgpack_open(self.info_filename) as f: info = f.next() self.count = info['count'] self.colsByName = info['colsByName'] self.children = [] for child_bounds in self.bounds.get_children(): child = QuadtreeNode(self.tree, child_bounds) if os.path.exists(child.info_filename): self.children.append(child) child.load() if not self.children: self.children = None
def __init__(self, filename, __bare__ = False, **kw): self.filename = filename for key, value in kw.iteritems(): setattr(self, key, value) if __bare__: return self.root = quad_tree_node.QuadtreeNode(self) print "Loading data..." with utils.msgpack_open(self.root.source_filename, "w") as outf: with gpsdio.open(filename) as f: for row in f: out_row = {} for key, value in row.iteritems(): if isinstance(value, datetime.datetime): value = float(value.strftime("%s")) * 1000.0 if not isinstance(value, (float, int, bool)): continue out_row[key] = value outf.write(out_row) self.root.count += 1
def generate_tile_from_source(self): with utils.msgpack_open(self.source_filename) as f: rows = list(f) self.write_tile([cluster_mod.Cluster.from_row(row) for row in rows])