class TestRectangleRange(TestBase): query_envelope = Envelope(-90.01, -80.01, 30.01, 40.01) loop_times = 5 def test_spatial_range_query(self): spatial_rdd = RectangleRDD(self.sc, inputLocation, offset, splitter, True, StorageLevel.MEMORY_ONLY) for i in range(self.loop_times): result_size = RangeQuery.SpatialRangeQuery(spatial_rdd, self.query_envelope, False, False).count() assert result_size == 193 assert RangeQuery.SpatialRangeQuery( spatial_rdd, self.query_envelope, False, False).take(10)[1].getUserData() is not None def test_spatial_range_query_using_index(self): spatial_rdd = RectangleRDD(self.sc, inputLocation, offset, splitter, True, StorageLevel.MEMORY_ONLY) spatial_rdd.buildIndex(IndexType.RTREE, False) for i in range(self.loop_times): result_size = RangeQuery.SpatialRangeQuery(spatial_rdd, self.query_envelope, False, True).count() assert result_size == 193 assert RangeQuery.SpatialRangeQuery(spatial_rdd, self.query_envelope, False, True).take(10)[1].getUserData()\ is not None
class TestPointRange(TestBase): loop_times = 5 query_envelope = Envelope(-90.01, -80.01, 30.01, 40.01) def test_spatial_range_query(self): spatial_rdd = PointRDD(self.sc, input_location, offset, splitter, False) for i in range(self.loop_times): result_size = RangeQuery.\ SpatialRangeQuery(spatial_rdd, self.query_envelope, False, False)\ .count() assert result_size == 2830 assert RangeQuery.SpatialRangeQuery( spatial_rdd, self.query_envelope, False, False).take(10)[1].\ getUserData() is not None def test_spatial_range_query_using_index(self): spatial_rdd = PointRDD(self.sc, input_location, offset, splitter, False) spatial_rdd.buildIndex(IndexType.RTREE, False) for i in range(self.loop_times): result_size = RangeQuery.\ SpatialRangeQuery(spatial_rdd, self.query_envelope, False, False)\ .count() assert result_size == 2830 assert RangeQuery.SpatialRangeQuery( spatial_rdd, self.query_envelope, False, False).take(10)[1].\ getUserData() is not None
def SpatialRangeQuery(self, spatialRDD: SpatialRDD, rangeQueryWindow: Envelope, considerBoundaryIntersection: bool, usingIndex: bool): """ :param spatialRDD: :param rangeQueryWindow: :param considerBoundaryIntersection: :param usingIndex: :return: """ jvm = spatialRDD._jvm sc = spatialRDD._sc jvm_envelope = rangeQueryWindow.create_jvm_instance(jvm) srdd = jvm.\ RangeQuery.SpatialRangeQuery( spatialRDD._srdd, jvm_envelope, considerBoundaryIntersection, usingIndex ) serlialized = jvm.GeoSerializerData.serializeToPython(srdd) return RDD(serlialized, sc, GeoSparkPickler())
class TestPolygonRange(TestBase): loop_times = 5 query_envelope = Envelope(-85.01, -60.01, 34.01, 50.01) def test_spatial_range_query(self): spatial_rdd = PolygonRDD(self.sc, input_location, splitter, True, StorageLevel.MEMORY_ONLY) for i in range(self.loop_times): result_size = RangeQuery.\ SpatialRangeQuery(spatial_rdd, self.query_envelope, False, False).count() assert result_size == 704 assert RangeQuery.SpatialRangeQuery( spatial_rdd, self.query_envelope, False, False).take(10)[0].getUserData() is not None def test_spatial_range_query_using_index(self): spatial_rdd = PolygonRDD(self.sc, input_location, splitter, True, StorageLevel.MEMORY_ONLY) spatial_rdd.buildIndex(IndexType.RTREE, False) for i in range(self.loop_times): result_size = RangeQuery.\ SpatialRangeQuery(spatial_rdd, self.query_envelope, False, False).count() assert result_size == 704 assert RangeQuery.SpatialRangeQuery( spatial_rdd, self.query_envelope, False, False).take(10)[0].getUserData() is not None
class TestRectangleKNN(TestBase): query_envelope = Envelope(-90.01, -80.01, 30.01, 40.01) loop_times = 5 query_point = Point(-84.01, 34.01) top_k = 100 def test_spatial_knn_query(self): rectangle_rdd = RectangleRDD(self.sc, inputLocation, offset, splitter, True) for i in range(self.loop_times): result = KNNQuery.SpatialKnnQuery(rectangle_rdd, self.query_point, self.top_k, False) assert result.__len__() > -1 assert result[0].getUserData() is not None def test_spatial_knn_query_using_index(self): rectangle_rdd = RectangleRDD(self.sc, inputLocation, offset, splitter, True) rectangle_rdd.buildIndex(IndexType.RTREE, False) for i in range(self.loop_times): result = KNNQuery.SpatialKnnQuery(rectangle_rdd, self.query_point, self.top_k, False) assert result.__len__() > -1 assert result[0].getUserData() is not None def test_spatial_knn_query_correctness(self): rectangle_rdd = RectangleRDD(self.sc, inputLocation, offset, splitter, True) result_no_index = KNNQuery.SpatialKnnQuery(rectangle_rdd, self.query_point, self.top_k, False) rectangle_rdd.buildIndex(IndexType.RTREE, False) result_with_index = KNNQuery.SpatialKnnQuery(rectangle_rdd, self.query_point, self.top_k, True) sorted_result_no_index = sorted( result_no_index, key=lambda geo_data: distance_sorting_functions( geo_data, self.query_point)) sorted_result_with_index = sorted( result_with_index, key=lambda geo_data: distance_sorting_functions( geo_data, self.query_point)) difference = 0 for x in range(self.top_k): difference += sorted_result_no_index[x].geom.distance( sorted_result_with_index[x].geom) assert difference == 0
def boundaryEnvelope(self) -> Envelope: """ :return: """ if not self._is_analyzed: raise TypeError("Please use analyze before") java_boundary_envelope = get_field(self._srdd, "boundaryEnvelope") return Envelope.from_jvm_instance(java_boundary_envelope)
def boundary(self) -> Envelope: """ :return: """ jvm_boundary = self._srdd.boundary() envelope = Envelope.from_jvm_instance(jvm_boundary) return envelope
def grids(self) -> Optional[List[Envelope]]: """ Returns grids for SpatialRDD, it is a list of Envelopes. >> spatial_rdd.grids >> [Envelope(minx=10.0, maxx=12.0, miny=10.0, maxy=12.0)] :return: """ jvm_grids = self.jvm_grids.jgrid if jvm_grids: number_of_grids = jvm_grids.size() envelopes = [ Envelope.from_jvm_instance(jvm_grids[index]) for index in range(number_of_grids) ] return envelopes else: return None
def test_boundary_envelope(self): spatial_rdd = self.create_spatial_rdd() spatial_rdd.analyze() assert Envelope( minx=-173.120769, maxx=-84.965961, miny=30.244859, maxy=71.355134) == spatial_rdd.boundaryEnvelope
def test_boundary(self): spatial_rdd = self.create_spatial_rdd() envelope = spatial_rdd.boundary() assert envelope == Envelope(minx=-173.120769, maxx=-84.965961, miny=30.244859, maxy=71.355134)
from tests.test_base import TestBase from tests.tools import tests_path inputLocation = os.path.join(tests_path, "resources/zcta510-small.csv") queryWindowSet = os.path.join(tests_path, "resources/zcta510-small.csv") offset = 0 splitter = FileDataSplitter.CSV gridType = "rtree" indexType = "rtree" numPartitions = 11 distance = 0.001 queryPolygonSet = os.path.join(tests_path, "resources/primaryroads-polygon.csv") inputCount = 3000 inputBoundary = Envelope(minx=-171.090042, maxx=145.830505, miny=-14.373765, maxy=49.00127) matchCount = 17599 matchWithOriginalDuplicatesCount = 17738 class TestRectangleRDD(TestBase): def test_constructor(self): spatial_rdd = RectangleRDD(sparkContext=self.sc, InputLocation=inputLocation, Offset=offset, splitter=splitter, carryInputData=True, partitions=numPartitions, newLevel=StorageLevel.MEMORY_ONLY)
from tests.tools import tests_path resource_folder = "resources" point_rdd_input_location = os.path.join(tests_path, resource_folder, "arealm-small.csv") point_rdd_splitter = FileDataSplitter.CSV point_rdd_index_type = IndexType.RTREE point_rdd_num_partitions = 5 point_rdd_offset = 1 knn_query_point = Point(-84.01, 34.01) range_query_window = Envelope(-90.01, -80.01, 30.01, 40.01) join_query_partitionin_type = GridType.QUADTREE each_query_loop_times = 1 class TestSpatialRDD(TestBase): def test_empty_constructor_test(self): object_rdd = PointRDD(sparkContext=self.sc, InputLocation=point_rdd_input_location, Offset=point_rdd_offset, splitter=point_rdd_splitter, carryInputData=False) object_rdd_copy = PointRDD() object_rdd_copy.rawJvmSpatialRDD = object_rdd.rawJvmSpatialRDD object_rdd_copy.analyze()
from geo_pyspark.core.spatialOperator import KNNQuery from tests.test_base import TestBase from tests.tools import tests_path, distance_sorting_functions inputLocation = os.path.join(tests_path, "resources/zcta510-small.csv") queryWindowSet = os.path.join(tests_path, "resources/zcta510-small.csv") offset = 0 splitter = FileDataSplitter.CSV gridType = "rtree" indexType = "rtree" numPartitions = 11 distance = 0.001 queryPolygonSet = os.path.join(tests_path, "resources/primaryroads-polygon.csv") inputCount = 3000 inputBoundary = Envelope(-171.090042, 145.830505, -14.373765, 49.00127) matchCount = 17599 matchWithOriginalDuplicatesCount = 17738 class TestRectangleKNN(TestBase): query_envelope = Envelope(-90.01, -80.01, 30.01, 40.01) loop_times = 5 query_point = Point(-84.01, 34.01) top_k = 100 def test_spatial_knn_query(self): rectangle_rdd = RectangleRDD(self.sc, inputLocation, offset, splitter, True) for i in range(self.loop_times):
from geo_pyspark.core.spatialOperator import RangeQuery from tests.test_base import TestBase from tests.tools import tests_path input_location = os.path.join(tests_path, "resources/arealm-small.csv") queryWindowSet = os.path.join("zcta510-small.csv") offset = 1 splitter = FileDataSplitter.CSV gridType = "rtree" indexType = "rtree" numPartitions = 11 distance = 0.01 queryPolygonSet = "primaryroads-polygon.csv" inputCount = 3000 inputBoundary = Envelope(minx=-173.120769, maxx=-84.965961, miny=30.244859, maxy=71.355134) rectangleMatchCount = 103 rectangleMatchWithOriginalDuplicatesCount = 103 polygonMatchCount = 472 polygonMatchWithOriginalDuplicatesCount = 562 class TestPointRange(TestBase): loop_times = 5 query_envelope = Envelope(-90.01, -80.01, 30.01, 40.01) def test_spatial_range_query(self): spatial_rdd = PointRDD(self.sc, input_location, offset, splitter, False) for i in range(self.loop_times):
import os from geo_pyspark.core.enums import FileDataSplitter from geo_pyspark.core.geom_types import Envelope from tests.tools import tests_path input_location = os.path.join(tests_path, "resources/arealm-small.csv") query_window_set = os.path.join("zcta510-small.csv") offset = 1 splitter = FileDataSplitter.CSV grid_type = "rtree" index_type = "rtree" num_partitions = 11 distance = 0.01 query_polygon_set = "primaryroads-polygon.csv" input_count = 3000 input_boundary = Envelope( minx=-173.120769, maxx=-84.965961, miny=30.244859, maxy=71.355134 ) rectangle_match_count = 103 rectangle_with_original_duplicates_count = 103 polygon_match_count = 472 polygon_match_with_original_duplicates_count = 562 transformed_envelope = Envelope(14313844.29433424, 16587207.463797055, 942450.5989896542, 6697987.652517834) crs_point_test = os.path.join(tests_path, "resources/crs-test-point.csv") crs_envelope = Envelope(26.992172, 71.35513400000001, -179.147236, 179.475569) crs_envelope_transformed = Envelope(-5446655.086752228, 1983668.382852457, 534241.8622328962, 6143259.02554563)
def test_jvm_envelope(self): envelope = Envelope(0.0, 5.0, 0.0, 5.0) jvm_instance = envelope.create_jvm_instance(self.spark.sparkContext._jvm) envelope_area = jvm_instance.getArea() assert envelope_area == 25.0, f"Expected area to be equal 25 but {envelope_area} was found"
def test_get_envelope_internal(self): point = Point(0.0, 0.0) circle = Circle(point, 0.1) assert Envelope(-0.1, 0.1, -0.1, 0.1) == circle.getEnvelopeInternal()
import os from shapely.geometry import Point from geo_pyspark.core.enums import FileDataSplitter, GridType, IndexType from geo_pyspark.core.geom_types import Envelope from tests.tools import tests_path input_location = os.path.join(tests_path, "resources/crs-test-point.csv") offset = 0 splitter = FileDataSplitter.CSV grid_type = GridType.RTREE index_type = IndexType.RTREE num_partitions = 11 distance = 0.01 input_location_query_polygon = os.path.join(tests_path, "resources/crs-test-polygon.csv") loop_times = 5 query_envelope = Envelope(30.01, 40.01, -90.01, -80.01) query_point = Point(34.01, -84.01) top_k = 100
from geo_pyspark.core.enums import FileDataSplitter, IndexType from geo_pyspark.core.geom_types import Envelope from tests.tools import tests_path input_location = os.path.join(tests_path, "resources/primaryroads-polygon.csv") query_window_set = os.path.join(tests_path, "resources/zcta510-small.csv") offset = 0 splitter = FileDataSplitter.CSV grid_type = "rtree" index_type = "rtree" num_partitions = 5 distance = 0.01 input_location_query_polygon = os.path.join(tests_path, "resources/crs-test-polygon.csv") query_polygon_count = 13361 query_envelope = Envelope(14313844.29433424, 16802290.85383074, 942450.5989896542, 8631908.270651951) query_polygon_set = os.path.join(tests_path, "resources/primaryroads-polygon.csv") input_location_geo_json = os.path.join(tests_path, "resources/testPolygon.json") input_location_wkt = os.path.join(tests_path, "resources/county_small.tsv") input_location_wkb = os.path.join(tests_path, "resources/county_small_wkb.tsv") input_count = 3000 input_boundary = Envelope(minx=-158.104182, maxx=-66.03575, miny=17.986328, maxy=48.645133) contains_match_count = 6941 contains_match_with_original_duplicates_count = 9334 intersects_match_count = 24323 intersects_match_with_original_duplicates_count = 32726 polygon_rdd_input_location = os.path.join(tests_path, "resources/primaryroads-polygon.csv") polygon_rdd_splitter = FileDataSplitter.CSV polygon_rdd_index_type = IndexType.RTREE polygon_rdd_num_partitions = 5
import os from geo_pyspark.core.enums import FileDataSplitter from geo_pyspark.core.geom_types import Envelope from tests.tools import tests_path input_location = os.path.join(tests_path, "resources/primaryroads-linestring.csv") query_window_set = os.path.join(tests_path, "resources/zcta510-small.csv") offset = 0 splitter = FileDataSplitter.CSV grid_type = "rtree" index_type = "rtree" num_partitions = 5 distance = 0.01 query_polygon_set = os.path.join(tests_path, "resources/primaryroads-polygon.csv") input_count = 3000 input_boundary = Envelope(minx=-123.393766, maxx=-65.648659, miny=17.982169, maxy=49.002374) input_boundary_2 = Envelope(minx=-123.393766, maxx=-65.649956, miny=17.982169, maxy=49.002374) match_count = 535 match_with_origin_with_duplicates_count = 875 transformed_envelope = Envelope(14313844.29433424, 16791709.85358734, 942450.5989896542, 8474779.278028419) transformed_envelope_2 = Envelope(14313844.29433424, 16791709.85358734, 942450.5989896542, 8474645.488977494)