def prepare_rdd(self, object_rdd: SpatialRDD, window_rdd: SpatialRDD,
grid_type: GridType):
object_rdd.analyze()
window_rdd.analyze()
object_rdd.rawSpatialRDD.repartition(4)
object_rdd.spatialPartitioning(grid_type)
object_rdd.buildIndex(IndexType.RTREE, True)
window_rdd.spatialPartitioning(object_rdd.getPartitioner())
def compare_spatial_rdd(self, spatial_rdd: SpatialRDD,
envelope: Envelope) -> bool:
spatial_rdd.analyze()
assert input_count == spatial_rdd.approximateTotalCount
assert envelope == spatial_rdd.boundaryEnvelope
return True
def test_to_rdd_from_dataframe(self):
spatial_df = self._create_spatial_point_table()
spatial_df.show()
jsrdd = Adapter.toRdd(spatial_df)
spatial_rdd = SpatialRDD(self.sc)
spatial_rdd.rawJvmSpatialRDD = jsrdd
spatial_rdd.analyze()
assert spatial_rdd.approximateTotalCount == 121960
assert spatial_rdd.boundaryEnvelope == Envelope(
-179.147236, 179.475569, -14.548699, 71.35513400000001)
def test_read_csv_point_into_spatial_rdd_by_passing_coordinates(self):
df = self.spark.read.format("csv").\
option("delimiter", ",").\
option("header", "false").\
load(area_lm_point_input_location)
df.show()
df.createOrReplaceTempView("inputtable")
spatial_df = self.spark.sql(
"select ST_Point(cast(inputtable._c0 as Decimal(24,20)),cast(inputtable._c1 as Decimal(24,20))) as arealandmark from inputtable"
)
spatial_df.show()
spatial_df.printSchema()
spatial_rdd = SpatialRDD(self.spark.sparkContext)
spatial_rdd.rawJvmSpatialRDD = Adapter.toRdd(spatial_df)
spatial_rdd.analyze()
assert (Adapter.toDf(spatial_rdd, self.spark).columns.__len__() == 1)
Adapter.toDf(spatial_rdd, self.spark).show()
def compare_count(self, spatial_rdd: SpatialRDD, cnt: int,
envelope: Envelope):
spatial_rdd.analyze()
assert cnt == spatial_rdd.approximateTotalCount
assert envelope == spatial_rdd.boundaryEnvelope