Example #1
0
  def setup_rtree(self, ndim, clusters=None):
    if self._rtree:
        return self._rtree

    self._ndim = ndim
    if not ndim:
        class k(object):
            def __init__(self, graph):
                self.graph = graph
            def insert(self, *args, **kwargs):
                pass
            def delete(self, *args, **kwargs):
                pass
            def intersection(self, *args, **kwargs):
                return xrange(len(self.graph.clusters))
        self._rtree = k(self)
        return self._rtree


    p = RProp()
    p.dimension = max(2, ndim)
    p.dat_extension = 'data'
    p.idx_extension = 'index'

    if clusters:
        gen_func = ((i, self.bbox_rtree(c, enlarge=0.005), None) for i, c in enumerate(clusters))
        self._rtree = RTree(gen_func, properties=p)
    else:
        self._rtree = RTree(properties=p)
    return self._rtree
Example #2
0
def get_sindex(gdf):
    """Get or build an R-Tree spatial index.

    Particularly useful for geopandas<0.2.0;>0.7.0;0.9.0
    """
    sindex = None
    if (hasattr(gdf, '_rtree_sindex')):
        return getattr(gdf, '_rtree_sindex')
    if (isinstance(gdf, geopandas.GeoDataFrame)
            and hasattr(gdf.geometry, 'sindex')):
        sindex = gdf.geometry.sindex
    elif isinstance(gdf, geopandas.GeoSeries) and hasattr(gdf, 'sindex'):
        sindex = gdf.sindex
    if sindex is not None:
        if (hasattr(sindex, "nearest")
                and sindex.__class__.__name__ != "PyGEOSSTRTreeIndex"):
            # probably rtree.index.Index
            return sindex
        else:
            # probably PyGEOSSTRTreeIndex but unfortunately, 'nearest'
            # with 'num_results' is required
            sindex = None
    if rtree and len(gdf) >= rtree_threshold:
        # Manually populate a 2D spatial index for speed
        sindex = Index()
        # slow, but reliable
        for idx, item in enumerate(gdf.bounds.itertuples()):
            sindex.add(idx, item[1:])
        # cache the index for later
        setattr(gdf, '_rtree_sindex', sindex)
    return sindex
Example #3
0
 def read_airspace(self, airspace):
     index = Index()
     with open(airspace, 'r') as f:
         reader = openair.Reader(f)
         for record, error in reader:
             if error:
                 logging.warning(
                     f'line {error.lineno} of {os.path.basename(airspace)} - {error}'
                 )
             else:
                 try:
                     zone = Airspace(record)
                     if not self.agl_validable and (zone.ground_floor
                                                    or zone.ground_ceiling):
                         logging.warning(
                             f'{zone.name} will not be checked because ground altitude of flight could not be retrieved.'
                         )
                     else:
                         if zone.bounds:
                             index.insert(id(zone), zone.bounds, obj=zone)
                 except KeyError:
                     logging.warning(
                         f'line {reader.reader.lineno} of {os.path.basename(airspace)} - error in previous record'
                     )
     return index
Example #4
0
def create_spatial_index(shape_dict):
    print >> sys.stderr, 'Making spatial index...',
    spatial_index = Index()
    for index, (blockid, shape) in enumerate(shape_dict.iteritems()):
        spatial_index.insert(index, shape.bounds, obj=blockid)
    print >> sys.stderr, 'done.'
    return spatial_index
Example #5
0
def create_spatial_index(shape_dict):
    print >> sys.stderr, 'Making spatial index...',
    spatial_index = Index()
    for index, (blockid, shape) in enumerate(shape_dict.iteritems()):
        spatial_index.insert(index, shape.bounds, obj=blockid)
    print >> sys.stderr, 'done.'
    return spatial_index
Example #6
0
    def setup_rtree(self, ndim, clusters=None):
        if self._rtree:
            return self._rtree

        self._ndim = ndim
        if not ndim:

            class k(object):
                def __init__(self, graph):
                    self.graph = graph

                def insert(self, *args, **kwargs):
                    pass

                def delete(self, *args, **kwargs):
                    pass

                def intersection(self, *args, **kwargs):
                    return xrange(len(self.graph.clusters))

            self._rtree = k(self)
            return self._rtree

        p = RProp()
        p.dimension = max(2, ndim)
        p.dat_extension = 'data'
        p.idx_extension = 'index'

        if clusters:
            gen_func = ((i, self.bbox_rtree(c, enlarge=0.005), None)
                        for i, c in enumerate(clusters))
            self._rtree = RTree(gen_func, properties=p)
        else:
            self._rtree = RTree(properties=p)
        return self._rtree
Example #7
0
class GeneticLabeler(BaseGeneticLabeler):
    def __init__(self, points, bounding_box):
        BaseGeneticLabeler.__init__(self, points, bounding_box)
        self.build_index()

    def build_index(self):
        label_candidates = []
        for p in self.points:
            label_candidates.extend(p.label_candidates)
        self.items = []
        self.items.extend(label_candidates)
        self.items.extend(self.points)
        self.items.extend(self.bounding_box.border_config)

        self.idx = Index()
        for i, item in enumerate(self.items):
            item.index = i
            self.idx.insert(i, item.box)

    def evaluate_fitness(self, individual):
        penalty = 0
        for lpid, pos in enumerate(individual):
            self.labeled_points[lpid].label_candidates[pos].select()

        for lpid, lcid in enumerate(individual):
            lp = self.labeled_points[lpid]
            lc = lp.label_candidates[lcid]

            penalty += evaluate_label(lc, self.items, self.idx, selected_only=True)
        return (-penalty,)
Example #8
0
 def __init__(self, resolution, basename=None, overwrite=False):
     # type: (float) -> None
     self._res = resolution
     self._cnt = 0
     if basename is None:
         self._index = Index(interleaved=True)
     else:
         p = Property(overwrite=overwrite)
         self._index = Index(basename, interleaved=True, properties=p)
Example #9
0
 def _construct_index(self):
     """!
     Separate the index construction from the constructor, allowing a GUI override
     """
     self.index = Index()
     for i, j, k in self.ikle:
         t = Polygon([self.points[i], self.points[j], self.points[k]])
         self.triangles[i, j, k] = t
         self.index.insert(i, t.bounds, obj=(i, j, k))
Example #10
0
 def _construct_index(self, iter_pbar):
     """!
     Separate the index construction from the constructor, allowing a GUI override
     @param iter_pbar: iterable progress bar
     """
     self.index = Index()
     for i, j, k in iter_pbar(self.ikle, unit='elements'):
         t = Polygon([self.points[i], self.points[j], self.points[k]])
         self.triangles[i, j, k] = t
         self.index.insert(i, t.bounds, obj=(i, j, k))
Example #11
0
def compute_indicatormatrix(orig,
                            dest,
                            orig_proj='latlong',
                            dest_proj='latlong'):
    """
    Compute the indicatormatrix

    The indicatormatrix I[i,j] is a sparse representation of the ratio
    of the area in orig[j] lying in dest[i], where orig and dest are
    collections of polygons, i.e.

    A value of I[i,j] = 1 indicates that the shape orig[j] is fully
    contained in shape dest[j].

    Note that the polygons must be in the same crs.

    Parameters
    ---------
    orig : Collection of shapely polygons
    dest : Collection of shapely polygons

    Returns
    -------
    I : sp.sparse.lil_matrix
      Indicatormatrix
    """

    dest = reproject_shapes(dest, dest_proj, orig_proj)
    indicator = sp.sparse.lil_matrix((len(dest), len(orig)), dtype=np.float)

    try:
        from rtree.index import Index

        idx = Index()
        for j, o in enumerate(orig):
            idx.insert(j, o.bounds)

        for i, d in enumerate(dest):
            for j in idx.intersection(d.bounds):
                o = orig[j]
                area = d.intersection(o).area
                indicator[i, j] = area / o.area

    except ImportError:
        logger.warning(
            "Rtree is not available. Falling back to slower algorithm.")

        dest_prepped = list(map(prep, dest))

        for i, j in product(range(len(dest)), range(len(orig))):
            if dest_prepped[i].intersects(orig[j]):
                area = dest[i].intersection(orig[j]).area
                indicator[i, j] = area / orig[j].area

    return indicator
Example #12
0
    def build_index(self):
        label_candidates = []
        for p in self.points:
            label_candidates.extend(p.label_candidates)
        self.items = []
        self.items.extend(label_candidates)
        self.items.extend(self.points)
        self.items.extend(self.bounding_box.border_config)

        self.idx = Index()
        for i, item in enumerate(self.items):
            item.index = i
            self.idx.insert(i, item.box)
Example #13
0
class StreetIndex(object):
    def __init__(self, streets_file):
        self.idx = Index()
        with open(streets_file) as f:
            for line in f.readlines():
                street = json.loads(line)
                street_id = street['properties']['id']
                street_shape = asShape(street['geometry'])
                for i in range(len(street_shape.geoms)):
                    seg_id = self.encode_seg_id(i, street_id)
                    self.idx.insert(seg_id, street_shape.geoms[i].coords[0])
                    self.idx.insert(-seg_id, street_shape.geoms[i].coords[-1])

        self.bb_idx = Index()
        with open(streets_file) as f:
            for line in f.readlines():
                street = json.loads(line)
                street_id = int(street['properties']['id'])
                street_shape = asShape(street['geometry'])
                self.bb_idx.insert(street_id, list(street_shape.bounds))

    def encode_seg_id(self, i, street_id):
        return i * 1000000 + int(street_id)

    def decode_seg_id(self, seg_id):
        i = abs(seg_id) / 1000000
        return abs(seg_id) - i

    def find_nearest_street(self, shape):
        shape = asShape(shape['geometry'])
        shape_type = shape.geom_type
        if shape_type == 'Polygon' or shape_type == 'MultiPolygon':
            ref_point = (
                float(shape.centroid.coords.xy[0][0]),
                float(shape.centroid.coords.xy[1][0])
            )
        else:
            ref_point = (
                float(shape.coords.xy[0][0]),
                float(shape.coords.xy[1][0])
            )
        street_id = list(self.bb_idx.nearest(ref_point))[0]
        return str(street_id)

    def find_connected_street(self, street):
        street_id = int(street['properties']['id'])
        street_shape = asShape(street['geometry'])
        street_start = street_shape.geoms[0].coords[0]
        street_end = street_shape.geoms[-1].coords[-1]
        seg_ids = list(self.idx.intersection(street_start))
        seg_ids += list(self.idx.intersection(street_end))
        street_ids = set(map(self.decode_seg_id, seg_ids))
        if street_id in street_ids:
            street_ids.remove(street_id)
        return street_ids
Example #14
0
 def __init__(self, stype, bbox=None, **kwargs):
     """initializer"""
     ci = kwargs.pop('custom_index', None)
     self._filename = kwargs.pop('filename', None)
     self._bbox = bbox
     self._stype = stype.lower()
     self._df = None
     if ci and stype.lower() == 'custom':
         self._index = ci
     elif stype.lower() == 'quadtree' and bbox:
         self._index = QIndex(bbox=bbox)
     elif RIndex and stype.lower() == 'rtree':
         self._index = RIndex(self._filename)
     else:
         raise ValueError("Could not create the spatial index.")
Example #15
0
 def __init__(self, input_header, construct_index=False, iter_pbar=lambda x, unit: x):
     """!
     @param input_header <slf.Serafin.SerafinHeader>: input Serafin header
     @param construct_index <bool>: perform the index construction
     @param iter_pbar: iterable progress bar
     """
     self.x, self.y = input_header.x[:input_header.nb_nodes_2d], input_header.y[:input_header.nb_nodes_2d]
     self.ikle = input_header.ikle_2d - 1  # back to 0-based indexing
     self.triangles = {}
     self.nb_points = self.x.shape[0]
     self.nb_triangles = self.ikle.shape[0]
     self.points = np.stack([self.x, self.y], axis=1)
     if not construct_index:
         self.index = Index()
     else:
         self._construct_index(iter_pbar)
Example #16
0
class Mesh2D:
    """!
    The general representation of mesh in Serafin 2D.
    The basis for interpolation, volume calculations etc.
    """
    def __init__(self,
                 input_header,
                 construct_index=False,
                 iter_pbar=lambda x: x):
        """!
        @param input_header <slf.Serafin.SerafinHeader>: input Serafin header
        @param construct_index <bool>: perform the index construction
        @param iter_pbar: iterable progress bar
        """
        self.x, self.y = input_header.x[:input_header.
                                        nb_nodes_2d], input_header.y[:
                                                                     input_header
                                                                     .
                                                                     nb_nodes_2d]
        self.ikle = input_header.ikle_2d - 1  # back to 0-based indexing
        self.triangles = {}
        self.nb_points = self.x.shape[0]
        self.nb_triangles = self.ikle.shape[0]
        self.points = np.stack([self.x, self.y], axis=1)
        if not construct_index:
            self.index = Index()
        else:
            self._construct_index(iter_pbar)

    def _construct_index(self, iter_pbar):
        """!
        Separate the index construction from the constructor, allowing a GUI override
        @param iter_pbar: iterable progress bar
        """
        self.index = Index()
        for i, j, k in iter_pbar(self.ikle, unit='elements'):
            t = Polygon([self.points[i], self.points[j], self.points[k]])
            self.triangles[i, j, k] = t
            self.index.insert(i, t.bounds, obj=(i, j, k))

    def get_intersecting_elements(self, bounding_box):
        """!
        @brief Return the triangles in the mesh intersecting the bounding box
        @param bounding_box <tuple>: (left, bottom, right, top) of a 2d geometrical object
        @return <[tuple]>: The list of triangles (i,j,k) intersecting the bounding box
        """
        return list(self.index.intersection(bounding_box, objects='raw'))
Example #17
0
    def create_rtree(self, clusters):
        if not len(clusters[0].bbox[0]):
            class k(object):
                def intersection(self, foo):
                    return xrange(len(clusters))
            return k()

        ndim = len(clusters[0].bbox[0]) + 1
        p = RProp()
        p.dimension = ndim
        p.dat_extension = 'data'
        p.idx_extension = 'index'

        rtree = RTree(properties=p)
        for idx, c in enumerate(clusters):
            rtree.insert(idx, c.bbox[0] + (0,) + c.bbox[1] + (1,))
        return rtree
Example #18
0
    def test_tpr(self):
        # TODO : this freezes forever on some windows cloud builds
        if os.name == 'nt':
            return

        # Cartesians list for brute force
        objects = dict()
        tpr_tree = Index(properties=Property(type=RT_TPRTree))

        for operation, t_now, object_ in data_generator():
            if operation == "INSERT":
                tpr_tree.insert(object_.id, object_.get_coordinates())
                objects[object_.id] = object_
            elif operation == "DELETE":
                tpr_tree.delete(object_.id, object_.get_coordinates(t_now))
                del objects[object_.id]
            elif operation == "QUERY":
                tree_intersect = set(
                    tpr_tree.intersection(object_.get_coordinates()))

                # Brute intersect
                brute_intersect = set()
                for tree_object in objects.values():
                    x_low, y_low = tree_object.getXY(object_.start_time)
                    x_high, y_high = tree_object.getXY(object_.end_time)

                    if intersects(
                            x_low, y_low, x_high, y_high,  # Line
                            object_.x, object_.y, object_.dx, object_.dy):  # Rect
                        brute_intersect.add(tree_object.id)

                # Tree should match brute force approach
                assert tree_intersect == brute_intersect
Example #19
0
def demo_delete():
    seed = 1  # Seed for random points

    countries = get_countries()

    country_id_to_remove = 170  # United States of America
    country_uuids_to_remove = []  # Polygons' ids to remove from the index

    properties = Property()
    # properties.writethrough = True
    # properties.leaf_capacity = 1000
    # properties.fill_factor = 0.5
    index = Index(properties=properties)

    points_per_polygon = 1
    points = []

    # Inserts countries data to the index
    for i, (country_name, geometry) in enumerate(countries):
        for polygon in get_polygons(geometry):
            temp_uuid = uuid.uuid1().int
            index.insert(temp_uuid, polygon.bounds, country_name)

            if i == country_id_to_remove:
                # Saves index ids of the polygon to be removed later
                country_uuids_to_remove.append(temp_uuid)

            # Generates random points in every polygon and saves them
            random_points = gen_random_point(points_per_polygon, polygon, seed)
            points.append((country_name, random_points))

    # Checks every generated point has matches
    for (country_name, country_points) in points:
        for point in country_points:
            hits = list(index.intersection(point.bounds, objects=True))
            assert any(hit.object == country_name for hit in hits)

    # Remove geometry
    geometry = countries[country_id_to_remove][1]
    for i, polygon in enumerate(get_polygons(geometry)):
        index.delete(country_uuids_to_remove[i], polygon.bounds)

    points_missing = []

    # Checks (again) if every generated point has matches
    for (country_name, country_points) in points:
        for point in country_points:
            hits = list(index.intersection(point.bounds, objects=True))
            # Save any point without matches
            if not any(hit.object == country_name for hit in hits):
                points_missing.append(str(point) + " - " + country_name)

    # Print missing points
    for point in points_missing:
        print(point)
Example #20
0
def get_sindex(gdf):
    """Helper function to get or build a spatial index

    Particularly useful for geopandas<0.2.0
    """
    assert isinstance(gdf, geopandas.GeoDataFrame)
    has_sindex = hasattr(gdf, 'sindex')
    if has_sindex:
        sindex = gdf.geometry.sindex
    elif rtree and len(gdf) >= rtree_threshold:
        # Manually populate a 2D spatial index for speed
        sindex = Index()
        # slow, but reliable
        for idx, (segnum, row) in enumerate(gdf.bounds.iterrows()):
            sindex.add(idx, tuple(row))
    else:
        sindex = None
    return sindex
Example #21
0
    def create_rtree(self, clusters):
        if not len(clusters[0].bbox[0]):

            class k(object):
                def intersection(self, foo):
                    return xrange(len(clusters))

            return k()

        ndim = len(clusters[0].bbox[0]) + 1
        p = RProp()
        p.dimension = ndim
        p.dat_extension = 'data'
        p.idx_extension = 'index'

        rtree = RTree(properties=p)
        for idx, c in enumerate(clusters):
            rtree.insert(idx, c.bbox[0] + (0, ) + c.bbox[1] + (1, ))
        return rtree
Example #22
0
    def __init__(self, streets_file):
        self.idx = Index()
        with open(streets_file) as f:
            for line in f.readlines():
                street = json.loads(line)
                street_id = street['properties']['id']
                street_shape = asShape(street['geometry'])
                for i in range(len(street_shape.geoms)):
                    seg_id = self.encode_seg_id(i, street_id)
                    self.idx.insert(seg_id, street_shape.geoms[i].coords[0])
                    self.idx.insert(-seg_id, street_shape.geoms[i].coords[-1])

        self.bb_idx = Index()
        with open(streets_file) as f:
            for line in f.readlines():
                street = json.loads(line)
                street_id = int(street['properties']['id'])
                street_shape = asShape(street['geometry'])
                self.bb_idx.insert(street_id, list(street_shape.bounds))
Example #23
0
    def construct_rtree(self, clusters):
        if not len(clusters[0].bbox[0]):
            class k(object):
                def intersection(self, foo):
                    return xrange(len(clusters))
            return k()
        ndim = max(2, len(clusters[0].centroid))
        p = RProp()
        p.dimension = ndim
        p.dat_extension = 'data'
        p.idx_extension = 'index'

        rtree = RTree(properties=p)
        for idx, c in enumerate(clusters):
            box = c.bbox #self.scale_box(c.bbox)
            if ndim == 1:
                rtree.insert(idx, box[0] + [0] + box[1] + [1])
            else:
                rtree.insert(idx, box[0] + box[1])
        return rtree
Example #24
0
def snap_to_edge_position(gdf, points, k=3, rtree=None):
    """
    Snap given points in the plane to edges in GeoDataFrame of edges.

    Parameters
    ----------
    gdf : GeoDataframe
        The edges of spatial network as a Geodataframe.
    points : array of floats, shape (M, 2)
        The cartesian coordinates of the points to be snapped.
    k : integer, optional
        Number of nearest edges to consider.

    Returns
    -------
    nearest_edges : list of integers, length M
        Indices of nearest edges in the GeoDataframe.
    refdistances : list of floats, length M
        Linear referencing distances of points along nearest edge.
    """
    X, Y = points.T
    geom = gdf["geometry"]

    # If not passed, build the r-tree spatial index by position for subsequent iloc
    if rtree == None:
        rtree = RTreeIndex()
        for pos, bounds in enumerate(geom.bounds.values):
            rtree.insert(pos, bounds)

    # use r-tree to find possible nearest neighbors, one point at a time,
    # then minimize euclidean distance from point to the possible matches
    nearest_edges = list()
    refdistances = list()
    for xy in zip(X, Y):
        p = Point(xy)
        dists = geom.iloc[list(rtree.nearest(xy, num_results=k))].distance(p)
        ne = geom[dists.idxmin()]
        nearest_edges.append(dists.idxmin())
        refdistances.append(ne.project(p))

    return nearest_edges, refdistances
Example #25
0
def get_rtree(geometries, fp):
    fp = fp.as_posix()
    if not os.path.exists(fp + '.idx'):
        # Populate R-tree index with bounds of geometries
        print('Populate {} tree'.format(fp))
        idx = Index(fp)
        for i, geo in enumerate(geometries):
            idx.insert(i, geo.bounds)
        idx.close()

    return {'rtree': Index(fp), 'geometries': geometries}
Example #26
0
 def build_rtree(self):
     '''
     Construct an R-tree for the domain. This may reduce the
     computational complexity of the methods `intersection_count`,
     `contains`, `orient_simplices`, and `snap`.
     '''
     # create a bounding box for each simplex and add those
     # bounding boxes to the R-tree
     if self.rtree is not None:
         # do nothing because the R-tree already exists
         logger.debug('R-tree already exists')
         return
         
     smp_min = self.vertices[self.simplices].min(axis=1)
     smp_max = self.vertices[self.simplices].max(axis=1)
     bounds = np.hstack((smp_min, smp_max))
     
     p = Property()
     p.dimension = self.dim
     self.rtree = Index(properties=p)
     for i, bnd in enumerate(bounds):
         self.rtree.add(i, bnd)
Example #27
0
    def build_cache(self):
        label_candidates = []
        for p in self.points:
            label_candidates.extend(p.label_candidates)
        items = []
        items.extend(label_candidates)
        items.extend(self.points)
        items.extend(self.bounding_box.border_config)

        idx = Index()
        for i, item in enumerate(items):
            item.index = i
            idx.insert(i, item.box)

        for lc in label_candidates:
            lc.penalty = POSITION_WEIGHT * lc.position
            lc.label_penalties = [0 for i in range(len(label_candidates))]
            intersecting_item_ids = idx.intersection(lc.box)
            bbox_counted = False

            for item_id in intersecting_item_ids:
                item = items[item_id]

                if item == lc or item == lc.point:
                    continue

                if isinstance(item, Label):
                    if lc.point == item.point:
                        continue
                    else:
                        lc.label_penalties[item.index] = item.overlap(lc)
                        continue

                if isinstance(item, BoundingBoxBorder):
                    if bbox_counted:
                        continue
                    bbox_counted = True

                lc.penalty += item.overlap(lc)
Example #28
0
class PolyStore(object):
    def __init__(self):
        self.index = Index()

    def load_from_shapefile(self, sf):
        self.shapes = sf.shapes()
        self.records = sf.records()
        for index, shape in enumerate(self.shapes):
            if len(shape.parts) > 1:
                print self.records[index], len(shape.parts)
            self.index.insert(index, shape.bbox)

    def get_shape_at_point(self, (x, y)):
        candidates = self.index.intersection((x, y, x, y))
        for candidate in candidates:
            shape = self.shapes[candidate]
            for i, part in enumerate(shape_to_parts_list(shape)):
                if Polygon(part).contains(Point(x, y)):
                    if i in SHAPE_LAND.get(self.records[candidate][4], []):
                        return '0'
                    return self.records[candidate][4]
        return None
Example #29
0
class PolyStore(object):
    def __init__(self):
        self.index = Index()

    def load_from_shapefile(self, sf):
        self.shapes = sf.shapes()
        self.records = sf.records()
        for index, shape in enumerate(self.shapes):
            if len(shape.parts) > 1:
                print self.records[index], len(shape.parts)
            self.index.insert(index, shape.bbox)

    def get_shape_at_point(self, (x, y)):
        candidates = self.index.intersection((x, y, x, y))
        for candidate in candidates:
            shape = self.shapes[candidate]
            for i, part in enumerate(shape_to_parts_list(shape)):
                if Polygon(part).contains(Point(x, y)):
                    if i in SHAPE_LAND.get(self.records[candidate][4], []):
                        return '0'
                    return self.records[candidate][4]
        return None
Example #30
0
def local_search(points, bounding_box, iterations):
    labeled_points = [p for p in points if p.text]

    items = []
    items.extend([p.label for p in labeled_points])
    items.extend(points)
    items.extend(bounding_box.border_config)

    idx = Index()
    for i, item in enumerate(items):
        item.index = i
        idx.insert(item.index, item.box)

    for i in range(iterations):
        for lp in labeled_points:
            best_candidate = None
            min_penalty = None
            for lc1 in lp.label_candidates:
                penalty = POSITION_WEIGHT * lc1.position

                # Check overlap with other labels and points
                intersecting_item_ids = idx.intersection(lc1.box)
                for item_id in intersecting_item_ids:
                    item = items[item_id]
                    if hasattr(item, "point") and lc1.point == item.point:
                        continue
                    penalty += item.overlap(lc1)

                if min_penalty is None or penalty < min_penalty:
                    min_penalty = penalty
                    best_candidate = lc1

            # Remove the old label from the index
            idx.delete(lp.label.index, lp.label.box)

            # Select the new label
            best_candidate.select()

            # Add the new label to the index and item list
            idx.insert(len(items), lp.label.box)
            items.append(lp.label)
Example #31
0
def evaluate_labels(labels, points, bounding_box):
    items = []
    items.extend(labels)
    items.extend(points)
    items.extend(bounding_box.border_config)

    t1 = time.clock()
    idx = Index()

    for i, item in enumerate(items):
        item.index = i
        idx.insert(i, item.box)

    t2 = time.clock()
    # print(f"Index creation: {t2-t1}")

    # Update penalties for overlap with other objects
    penalties = [evaluate_label(l, items, idx) for l in labels]

    t3 = time.clock()
    # print(f"Overlap checking: {t3 - t2}")

    print(f"Total time: {t3 - t1}")
    return penalties
Example #32
0
    def flush(self):
        """
        Saves the index to disk if a filename is given for an R-Tree Spatial Index.

        **This applies only to the R-Tree implementation of the spatial index.**

        :returns: Boolean

        """
        if hasattr(self._index, 'flush'):
            getattr(self._index, 'flush')()
        elif self._stype == 'rtree' and \
             self._filename:
            self._index.close()
            self._index = RIndex(self._filename)
        else:
            return False
        return True
Example #33
0
def make_index(shapes):
    """Creates an index for fast and efficient spatial queries.

    Args:
      shapes: shapely shapes to bulk-insert bounding boxes for into the spatial index.

    Returns:
      The spatial index created from the shape's bounding boxes.
    """

    # Todo: benchmark these for our use-cases
    prop = Property()
    prop.dimension = 2
    prop.leaf_capacity = 1000
    prop.fill_factor = 0.9

    def bounded():
        for i, shape in enumerate(shapes):
            yield (i, shape.bounds, None)

    return Index(bounded(), properties=prop)
Example #34
0
    def __init__(self, context: DyCleeContext):
        self.context = context

        self.dense_µclusters: Set[MicroCluster] = Set()
        self.semidense_µclusters: Set[MicroCluster] = Set()
        self.outlier_µclusters: Set[MicroCluster] = Set()
        self.long_term_memory: Set[MicroCluster] = Set()
        self.eliminated: Set[MicroCluster] = Set()

        self.next_µcluster_index: int = 0
        self.next_class_label: int = 0
        self.n_steps: int = 0
        self.last_partitioning_step: int = 0
        self.last_density_step: int = 0

        if self.context.maintain_rtree:
            p = RTreeProperty(dimension=self.context.n_features)
            self.rtree = RTreeIndex(properties=p)
            # This mapping is used to retrieve microcluster objects from their hashes
            # stored with their locations in the R*-tree
            self.µcluster_map: Optional[dict[int, MicroCluster]] = {}
        else:
            self.rtree = None
            self.µcluster_map = None
Example #35
0
class SpatialIndex():
    """

    A spatial index is a type of extended index that allows you to index a
    spatial column. A spatial column is a table column that contains data of a
    spatial data type.

    Spatial indexes help to improve spatial query performance on a dataframe.
    Identifying a feature, selecting features, and joining data all have better
    performace when using spatial indexing.


    ====================     ==================================================
    Arguement                Description
    --------------------     --------------------------------------------------
    stype                    Required String. This sets the type of spatial
                             index being used by the user. The current types of
                             spatial indexes are: custom, rtree and quadtree.
    --------------------     --------------------------------------------------
    bbox                     Optional Tuple. The extent of the spatial data as:
                             (xmin, ymin, xmax, ymax). This parameter is required
                             if a QuadTree Spatial Index is being used.

                             Example:
                             bbox=(-100, -50, 100, 50)
    --------------------     --------------------------------------------------
    filename                 Optional String. The name of the spatial index
                             file. This is only supported by rtree spatial
                             indexes. For large datasets an rtree index can be
                             saved to disk and used at a later time. If this is
                             not provided the r-tree index will be in-memory.
    --------------------     --------------------------------------------------
    custom_index             Optional Object. Sometimes QuadTree and Rtree
                             indexing is not enough. A custom spatial index class
                             can be giving to the SpatialIndex class and used
                             using encapsulation.  The custom index must have two
                             methods: `intersect` that excepts a tuple, and
                             `insert` which must accept an oid and a bounding
                             box. This object is required when `stype` of
                             'custom' is specified.
    ====================     ==================================================


    """
    _stype = None
    _bbox = None
    _index = None
    _df = None

    #----------------------------------------------------------------------
    def __init__(self, stype, bbox=None, **kwargs):
        """initializer"""
        ci = kwargs.pop('custom_index', None)
        self._filename = kwargs.pop('filename', None)
        self._bbox = bbox
        self._stype = stype.lower()
        self._df = None
        if ci and stype.lower() == 'custom':
            self._index = ci
        elif stype.lower() == 'quadtree' and bbox:
            self._index = QIndex(bbox=bbox)
        elif RIndex and stype.lower() == 'rtree':
            self._index = RIndex(self._filename)
        else:
            raise ValueError("Could not create the spatial index.")

    #----------------------------------------------------------------------
    def intersect(self, bbox):
        """
        Returns the spatial features that intersect the bbox

        :bbox: tuple - (xmin,ymin,xmax,ymax)

        :returns: list
        """
        if self._stype.lower() in ['rtree']:
            return list(self._index.intersection(bbox))
        elif self._stype.lower() in ['quadtree']:
            return list(self._index.intersect(bbox=bbox))
        else:
            return list(self._index.intersect(bbox))

    #----------------------------------------------------------------------
    def insert(self, oid, bbox):
        """
        Inserts the entry into the spatial index

        :oid: unique id
        :bbox: tuple - (xmin,ymin,xmax,ymax)
        """
        if self._index is None:
            raise Exception(("Could not insert into a spatial index because "
                             "it does not exist."))
        if self._stype == 'rtree' and \
           HASRTREE and \
           isinstance(self._index, RIndex):

            r = self._index.insert(id=oid, coordinates=bbox, obj=None)
            self.flush()
            return r
        elif self._stype.lower() == 'quadtree':
            return self._index.insert(item=oid, bbox=bbox)
        elif self._stype.lower() == 'custom':
            r = self._index.intersect(oid, bbox)
            self.flush()
            return r

    #----------------------------------------------------------------------
    def flush(self):
        """
        Saves the index to disk if a filename is given for an R-Tree Spatial Index.

        **This applies only to the R-Tree implementation of the spatial index.**

        :returns: Boolean

        """
        if hasattr(self._index, 'flush'):
            getattr(self._index, 'flush')()
        elif self._stype == 'rtree' and \
             self._filename:
            self._index.close()
            self._index = RIndex(self._filename)
        else:
            return False
        return True
Example #36
0
 def __init__(self):
     self.index = Index()
Example #37
0
class AdjacencyGraph(object):
    def __init__(self, clusters, partitions_complete=True):
        self.partitions_complete = partitions_complete
        self.graph = defaultdict(set)
        self.cid = 0
        self.clusters = []
        self.id2c = dict()
        self.c2id = dict()
        self._rtree = None  # internal datastructure
        self._ndim = None

        self.bulk_init(clusters)

    def to_json(self):
        data = {
                'clusters' : [c and c.__dict__ or None for c in self.clusters],
                'id2c' : [(key, c.__dict__) for key, c in self.id2c.items()],
                'c2id' : [(c.__dict__, val) for c, val in self.c2id.items()],
                'graph' : [(key.__dict__, [val.__dict__ for val in vals]) for key, vals in self.graph.itemsiter()],
                'cid' : self.cid,
                '_ndim' : self._ndim,
                '_rtreename' : 'BLAH'
                }
        return json.dumps(data)

    def from_json(self, encoded):
        data = json.loads(encoded)
        self.clusters = [c and Cluster.from_dict(c) or None for c in data['clusters']]
        self.id2c = dict([(key, Cluster.from_dict(val)) for key, val in data['id2c']])
        self.c2id = dict([(Cluster.from_dict(key), val) for key, val in data['c2id']])
        self.graph = dict([(Cluster.from_dict(key), map(Cluster.from_dict, vals)) for key, vals in data['graph']])
        self.cid = data['cid']
        self._ndim = data['_ndim']
        self._rtree = None

    def setup_rtree(self, ndim, clusters=None):
        if self._rtree:
            return self._rtree

        self._ndim = ndim
        if not ndim:
            class k(object):
                def __init__(self, graph):
                    self.graph = graph
                def insert(self, *args, **kwargs):
                    pass
                def delete(self, *args, **kwargs):
                    pass
                def intersection(self, *args, **kwargs):
                    return xrange(len(self.graph.clusters))
            self._rtree = k(self)
            return self._rtree
 

        p = RProp()
        p.dimension = max(2, ndim)
        p.dat_extension = 'data'
        p.idx_extension = 'index'

        if clusters:
            gen_func = ((i, self.bbox_rtree(c, enlarge=0.00001), None) for i, c in enumerate(clusters))
            self._rtree = RTree(gen_func, properties=p)
        else:
            self._rtree = RTree(properties=p)
        return self._rtree

    def bbox_rtree(self, cluster, enlarge=0.):
        bbox = cluster.bbox
        lower, higher = map(list, bbox)
        if self._ndim == 1:
            lower.append(0)
            higher.append(1)

        if enlarge != 1.:
            lower = [v - enlarge for v in lower]
            higher = [v + enlarge for v in higher]

        bbox = lower + higher
        return bbox

    def insert_rtree(self, idx, cluster):
        self.setup_rtree(len(cluster.bbox[0]))
        self._rtree.insert(idx,self.bbox_rtree(cluster))
        return cluster

    def remove_rtree(self, idx, cluster):
        self.setup_rtree(len(cluster.bbox[0]))
        self._rtree.delete(idx, self.bbox_rtree(cluster))
        return cluster

    def search_rtree(self, cluster):
        self.setup_rtree(len(cluster.bbox[0]))
        bbox = self.bbox_rtree(cluster, enlarge=0.00001)
        res = [self.clusters[idx] for idx in self._rtree.intersection(bbox)]
        return filter(bool, res)

    def bulk_init(self, clusters):
        if clusters:
            self.setup_rtree(len(clusters[0].bbox[0]), clusters)

        self.clusters.extend(clusters)
        for cid, c in enumerate(clusters):
            self.id2c[cid] = c
            self.c2id[c] = cid

        for idx, c in enumerate(clusters):
            for n in self.search_rtree(c):
                if self.c2id[n] <= idx: continue
                if c.discretes_contains(n) and box_completely_contained(c.bbox, n.bbox): continue
                if not c.adjacent(n, 0.8): continue
                self.graph[c].add(n)
                self.graph[n].add(c)



    def insert(self, cluster):
        if cluster in self.graph:
            return

        self.graph[cluster] = set()
        #for o in self.search_rtree(cluster):
        for o in self.graph.keys():
            if cluster == o:
                continue
            if cluster.adjacent(o, 0.8) or (volume(intersection_box(cluster.bbox, o.bbox)) > 0 and not cluster.contains(o)):
                self.graph[cluster].add(o)
                self.graph[o].add(cluster)
        

        cid = len(self.clusters)
        self.clusters.append(cluster)
        self.id2c[cid] = cluster
        self.c2id[cluster] = cid
        self.insert_rtree(cid, cluster)

    def remove(self, cluster):
        if cluster not in self.graph:
            return

        try:
            for neigh in self.graph[cluster]:
                if not neigh == cluster:
                    self.graph[neigh].remove(cluster)
        except:
            pdb.set_trace()
        del self.graph[cluster]

        cid = self.c2id[cluster]
        self.remove_rtree(cid, cluster)
        del self.c2id[cluster]
        del self.id2c[cid]
        self.clusters[cid] = None

    def neighbors(self, cluster):
        if not self.partitions_complete:
            return filter(bool, self.clusters)

        if cluster in self.graph:
            return self.graph[cluster]

        ret = set()
        intersects = self.search_rtree(cluster)
        for key in filter(cluster.adjacent, intersects):
            if box_completely_contained(key.bbox, cluster.bbox):
                continue
            ret.update(self.graph[key])
        return ret
Example #38
0
class AdjacencyVersion(object):

  def __init__(self, feature_mapper):
    #self.partitions_complete = partitions_complete
    self.cid = 0
    self.disc_idxs = {}
    self.feature_mapper = feature_mapper
    self.radius = .15
    self.metric = 'hamming'

    self._rtree = None  # internal datastructure
    self._ndim = None
    self.clusters = []
    self.id2c = dict()
    self.c2id = dict()

  def to_json(self):
    data = {
            'clusters' : [c and c.__dict__ or None for c in self.clusters],
            'id2c' : [(key, c.__dict__) for key, c in self.id2c.items()],
            'c2id' : [(c.__dict__, val) for c, val in self.c2id.items()],
            'cid' : self.cid,
            '_ndim' : self._ndim,
            '_rtreename' : 'BLAH'
            }
    return json.dumps(data)

  def from_json(self, encoded):
    data = json.loads(encoded)
    self.clusters = [c and Cluster.from_dict(c) or None for c in data['clusters']]
    self.id2c = dict([(key, Cluster.from_dict(val)) for key, val in data['id2c']])
    self.c2id = dict([(Cluster.from_dict(key), val) for key, val in data['c2id']])
    self.cid = data['cid']
    self._ndim = data['_ndim']
    self._rtree = None

  def setup_rtree(self, ndim, clusters=None):
    if self._rtree:
        return self._rtree

    self._ndim = ndim
    if not ndim:
        class k(object):
            def __init__(self, graph):
                self.graph = graph
            def insert(self, *args, **kwargs):
                pass
            def delete(self, *args, **kwargs):
                pass
            def intersection(self, *args, **kwargs):
                return xrange(len(self.graph.clusters))
        self._rtree = k(self)
        return self._rtree


    p = RProp()
    p.dimension = max(2, ndim)
    p.dat_extension = 'data'
    p.idx_extension = 'index'

    if clusters:
        gen_func = ((i, self.bbox_rtree(c, enlarge=0.005), None) for i, c in enumerate(clusters))
        self._rtree = RTree(gen_func, properties=p)
    else:
        self._rtree = RTree(properties=p)
    return self._rtree

  def bbox_rtree(self, cluster, enlarge=0.):
    cols = cluster.cols
    bbox = cluster.bbox
    lower, higher = map(list, bbox)
    if self._ndim == 1:
      lower.append(0)
      higher.append(1)

    if enlarge != 0:
      for idx, col in enumerate(cols):
        rng = enlarge * self.feature_mapper.ranges[col]
        lower[idx] -= rng
        higher[idx] += rng

    bbox = lower + higher
    return bbox

  def insert_rtree(self, idx, cluster):
    self.setup_rtree(len(cluster.bbox[0]))
    self._rtree.insert(idx,self.bbox_rtree(cluster))
    return cluster

  def remove_rtree(self, idx, cluster):
    self.setup_rtree(len(cluster.bbox[0]))
    self._rtree.delete(idx, self.bbox_rtree(cluster))
    return cluster

  def search_rtree(self, cluster):
    self.setup_rtree(len(cluster.bbox[0]))
    bbox = self.bbox_rtree(cluster, enlarge=0.01)
    return self._rtree.intersection(bbox)
    res = [self.clusters[idx] for idx in self._rtree.intersection(bbox)]
    return filter(bool, res)

  def bulk_init(self, clusters):
    if not clusters: return

    self.setup_rtree(len(clusters[0].bbox[0]), clusters)
    self.clusters = clusters
    for cid, c in enumerate(clusters):
      self.id2c[cid] = c
      self.c2id[c] = cid
    
    for dim in self.feature_mapper.attrs:
      Xs = []
      for cidx, c in enumerate(clusters):
        Xs.append(self.feature_mapper(c, dim))
      idx = NearestNeighbors(
          radius=self.radius, 
          algorithm='ball_tree', 
          metric=self.metric
      )
      self.disc_idxs[dim] = idx
      self.disc_idxs[dim].fit(np.array(Xs))

  def contains(self, cluster):
    return cluster in self.c2id
  
  def remove(self, cluster):
    if cluster in self.c2id:
      cid = self.c2id[cluster]
      self.remove_rtree(cid, cluster)
      del self.c2id[cluster]
      del self.id2c[cid]
      self.clusters[cid] = None
      return True
    return False


  def neighbors(self, cluster):
    ret = None
    for name, vals in cluster.discretes.iteritems():
      if name not in self.disc_idxs:
        return []
      vect = self.feature_mapper(cluster, name)
      index = self.disc_idxs[name]
      dists, idxs = index.radius_neighbors(vect, radius=self.radius)
      idxs = set(idxs[0].tolist())

      if ret is None:
        ret = idxs
      else:
        ret.intersection_update(idxs)
        #ret.update(idxs)
      if not ret: return []

    idxs = self.search_rtree(cluster)
    if ret is None:
      ret = set(idxs)
    else:
      ret.intersection_update(set(idxs))

    return filter(bool, [self.clusters[idx] for idx in ret])


  """
def main(input_dir, output_dir):
    formatter = logging.Formatter('%(asctime)s %(levelname)s [%(name)s]: %(message)s')
    handler = logging.StreamHandler(sys.stderr)
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    logger.setLevel(logging.INFO)

    city_names = []
    rtree = RTreeIndex()

    cities_filename = os.path.join(tempfile.gettempdir(), 'cities.json')

    subprocess.check_call(['wget', 'https://raw.githubusercontent.com/mapzen/metroextractor-cities/master/cities.json', '-O', cities_filename])

    all_cities = json.load(open(cities_filename))

    i = 0

    for k, v in all_cities['regions'].iteritems():
        for city, data in v['cities'].iteritems():
            bbox = data['bbox']
            rtree.insert(i, (float(bbox['left']), float(bbox['bottom']), float(bbox['right']), float(bbox['top'])))        
            city_names.append(city)
            i += 1

    files = {name: open(os.path.join(output_dir, 'cities', '{}.geojson'.format(name)), 'w') for name in city_names}

    planet = open(os.path.join(output_dir, 'planet.geojson'), 'w')
    planet_addresses_only = open(os.path.join(output_dir, 'planet_addresses_only.json'), 'w')

    i = 0
    seen = set()

    for url, canonical, venues in gen_venues(input_dir):
        domain = urlparse.urlsplit(url).netloc.strip('www.')
        for props in venues:
            lat = props.get('latitude')
            lon = props.get('longitude')
            props['canonical'] = canonical
            props['url'] = url
            street = props.get('street_address')
            name = props.get('name')
            planet_hash = hashlib.md5(u'|'.join((name, street, str(lat), str(lon), domain)).encode('utf-8')).digest()
            address_hash = hashlib.md5(u'|'.join((name, street, domain)).encode('utf-8')).digest()
            props['guid'] = props.get('guid', random_guid())
            venue = venue_to_geojson(props)
            if lat is not None and lon is not None:
                try:
                    lat = float(lat)
                    lon = float(lon)
                except Exception:
                    lat = None
                    lon = None
            if lat is not None and lon is not None and planet_hash not in seen:
                cities = list(rtree.intersection((lon, lat, lon, lat)))
                if cities:
                    for c in cities:
                        f = files[city_names[c]]
                    f.write(json.dumps(venue) + '\n')
                if planet_hash not in seen:
                    planet.write(json.dumps(venue) + '\n')
                    seen.add(planet_hash)
            if address_hash not in seen:
                planet_addresses_only.write(json.dumps(props) + '\n')
                seen.add(address_hash)
            i += 1
            if i % 1000 == 0 and i > 0:
                logger.info('did {}'.format(i))

    logger.info('Creating manifest files')

    manifest_files = []

    for k, v in all_cities['regions'].iteritems():
        for city, data in v['cities'].iteritems():
            f = files[city]
            if f.tell() == 0:
                f.close()
                os.unlink(os.path.join(output_dir, 'cities', '{}.geojson'.format(city)))
                continue

            bbox = data['bbox']
            lat = midpoint(float(bbox['top']), float(bbox['bottom']))
            lon = midpoint(float(bbox['left']), float(bbox['right']))

            manifest_files.append({'latitude': lat, 'longitude': lon, 'file': '{}.geojson'.format(city), 'name': city.replace('_', ', ').replace('-', ' ').title()})

    manifest = {'files': manifest_files}

    json.dump(manifest, open(os.path.join(output_dir, 'manifest.json'), 'w'))

    logger.info('Done!')
Example #40
0
def create_spatial_index(shape_dict):

    spatial_index = Index()
    for index, (name, shape) in enumerate(shape_dict.iteritems()):
        spatial_index.insert(index, shape.bounds, obj=name)
    return spatial_index
Example #41
0
 def __init__(self, *args):
     RTreeIndex.__init__(self, *args)
Example #42
0
 def __init__(self, *args):
     if not base.HAS_SINDEX:
         raise ImportError("SpatialIndex needs `rtree`")
     RTreeIndex.__init__(self, *args)