class ProjectedExtentSchemaTest(BaseTestClass):
projected_extents = [
{'epsg': 2004, 'extent': {'xmax': 1.0, 'xmin': 0.0, 'ymax': 1.0, 'ymin': 0.0}, 'proj4': None},
{'epsg': 2004, 'extent': {'xmax': 3.0, 'xmin': 1.0, 'ymax': 4.0, 'ymin': 2.0}, 'proj4': None},
{'epsg': 2004, 'extent': {'xmax': 7.0, 'xmin': 5.0, 'ymax': 8.0, 'ymin': 6.0}, 'proj4': None}]
sc = BaseTestClass.geopysc.pysc._jsc.sc()
ew = BaseTestClass.geopysc.pysc._jvm.geopyspark.geotrellis.tests.schemas.ProjectedExtentWrapper
tup = ew.testOut(sc)
java_rdd = tup._1()
ser = AvroSerializer(tup._2())
rdd = RDD(java_rdd, BaseTestClass.geopysc.pysc, AutoBatchedSerializer(ser))
collected = rdd.collect()
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.geopysc.pysc._gateway.close()
def result_checker(self, actual_pe, expected_pe):
for actual, expected in zip(actual_pe, expected_pe):
self.assertDictEqual(actual, expected)
def test_encoded_pextents(self):
encoded = self.rdd.map(lambda s: s)
actual_encoded = encoded.collect()
self.result_checker(actual_encoded, self.projected_extents)
def test_decoded_pextents(self):
self.result_checker(self.collected, self.projected_extents)
class SpatialKeySchemaTest(BaseTestClass):
expected_keys = {'col': 7, 'row': 3}
sc = BaseTestClass.geopysc.pysc._jsc.sc()
ew = BaseTestClass.geopysc.pysc._jvm.geopyspark.geotrellis.tests.schemas.SpatialKeyWrapper
tup = ew.testOut(sc)
java_rdd = tup._1()
ser = AvroSerializer(tup._2())
rdd = RDD(java_rdd, BaseTestClass.geopysc.pysc, AutoBatchedSerializer(ser))
collected = rdd.first()
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.geopysc.pysc._gateway.close()
def result_checker(self, actual_keys, expected_keys):
self.assertDictEqual(actual_keys, expected_keys)
def test_encoded_keyss(self):
encoded = self.rdd.map(lambda s: s)
actual_encoded = encoded.first()
self.result_checker(actual_encoded, self.expected_keys)
def test_decoded_extents(self):
self.assertDictEqual(self.collected, self.expected_keys)
class ByteTileSchemaTest(BaseTestClass):
tiles = [
{'data': np.array([0, 0, 1, 1]).reshape(2, 2), 'no_data_value': -128},
{'data': np.array([1, 2, 3, 4]).reshape(2, 2), 'no_data_value': -128},
{'data': np.array([5, 6, 7, 8]).reshape(2, 2), 'no_data_value': -128}
]
sc = BaseTestClass.geopysc.pysc._jsc.sc()
tw = BaseTestClass.geopysc.pysc._jvm.geopyspark.geotrellis.tests.schemas.ByteArrayTileWrapper
tup = tw.testOut(sc)
java_rdd = tup._1()
ser = AvroSerializer(tup._2(), AvroRegistry.tile_decoder, AvroRegistry.tile_encoder)
rdd = RDD(java_rdd, BaseTestClass.geopysc.pysc, AutoBatchedSerializer(ser))
collected = rdd.collect()
def test_encoded_tiles(self):
encoded = self.rdd.map(lambda s: AvroRegistry.tile_encoder(s))
actual_encoded = encoded.collect()
expected_encoded = [
{'bands': [{'cols': 2, 'rows': 2, 'cells': bytearray([0, 0, 1, 1]), 'noDataValue': -128}]},
{'bands': [{'cols': 2, 'rows': 2, 'cells': bytearray([1, 2, 3, 4]), 'noDataValue': -128}]},
{'bands': [{'cols': 2, 'rows': 2, 'cells': bytearray([5, 6, 7, 8]), 'noDataValue': -128}]}
]
for actual, expected in zip(actual_encoded, expected_encoded):
self.assertEqual(actual, expected)
def test_decoded_tiles(self):
for actual, expected in zip(self.collected, self.tiles):
self.assertTrue((actual['data'] == expected['data']).all())
def create_tuple_serializer(self, schema, key_type, value_type):
decoder = \
self.avroregistry.create_partial_tuple_decoder(key_type=key_type,
value_type=value_type)
encoder = \
self.avroregistry.create_partial_tuple_encoder(key_type=key_type,
value_type=value_type)
return AutoBatchedSerializer(AvroSerializer(schema, decoder, encoder))
class MultibandSchemaTest(BaseTestClass):
arr = np.array(bytearray([0, 0, 1, 1])).reshape(2, 2)
no_data = -128
arr_dict = {'data': arr, 'no_data_value': no_data}
band_dicts = [arr_dict, arr_dict, arr_dict]
bands = [arr, arr, arr]
multiband_tile = np.array(bands)
multiband_dict = {'data': multiband_tile, 'no_data_value': no_data}
sc = BaseTestClass.geopysc.pysc._jsc.sc()
mw = BaseTestClass.geopysc.pysc._jvm.geopyspark.geotrellis.tests.schemas.ArrayMultibandTileWrapper
tup = mw.testOut(sc)
java_rdd = tup._1()
ser = AvroSerializer(tup._2(),
AvroRegistry.tile_decoder,
AvroRegistry.tile_encoder)
rdd = RDD(java_rdd, BaseTestClass.geopysc.pysc, AutoBatchedSerializer(ser))
collected = rdd.collect()
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.geopysc.pysc._gateway.close()
def test_encoded_multibands(self):
encoded = self.rdd.map(lambda s: AvroRegistry.tile_encoder(s))
actual_encoded = encoded.collect()[0]
expected_encoded = AvroRegistry.tile_encoder(self.multiband_dict)
for actual, expected in zip(actual_encoded['bands'], expected_encoded['bands']):
self.assertEqual(actual, expected)
def test_decoded_multibands(self):
expected_multibands = [
self.multiband_dict,
self.multiband_dict,
self.multiband_dict
]
for actual, expected in zip(self.collected, expected_multibands):
self.assertTrue((actual['data'] == expected['data']).all())
class ExtentSchemaTest(BaseTestClass):
ew = BaseTestClass.geopysc._jvm.geopyspark.geotrellis.tests.schemas.ExtentWrapper
tup = ew.testOut(BaseTestClass.geopysc.sc)
java_rdd = tup._1()
ser = AvroSerializer(tup._2())
rdd = RDD(java_rdd, BaseTestClass.geopysc.pysc, AutoBatchedSerializer(ser))
collected = rdd.collect()
expected_extents = [{
"xmin": 0.0,
"ymin": 0.0,
"xmax": 1.0,
"ymax": 1.0
}, {
"xmin": 1.0,
"ymin": 2.0,
"xmax": 3.0,
"ymax": 4.0
}, {
"xmin": 5.0,
"ymin": 6.0,
"xmax": 7.0,
"ymax": 8.0
}]
@pytest.fixture(scope='class', autouse=True)
def tearDown(self):
yield
BaseTestClass.geopysc.pysc._gateway.close()
def result_checker(self, actual_result, expected_result):
for actual, expected in zip(actual_result, expected_result):
self.assertDictEqual(actual, expected)
def test_encoded_extents(self):
encoded = self.rdd.map(lambda s: s)
actual_encoded = encoded.collect()
self.result_checker(actual_encoded, self.expected_extents)
def test_decoded_extents(self):
self.result_checker(self.collected, self.expected_extents)
class KeyValueRecordSchemaTest(unittest.TestCase):
extents = [{
'xmin': 0,
'ymin': 0,
'xmax': 1,
'ymax': 1
}, {
'xmin': 1,
'ymin': 2,
'xmax': 3,
'ymax': 4
}, {
'xmin': 5,
'ymin': 6,
'xmax': 7,
'ymax': 8
}]
arrs = [{
'data': np.array(bytearray([0, 1, 2, 3, 4, 5])).reshape(3, 2),
'no_data_value': -128
}, {
'data': np.array(bytearray([0, 1, 2, 3, 4, 5])).reshape(2, 3),
'no_data_value': -128
}, {
'data': np.array(bytearray([0, 1, 2, 3, 4, 5])).reshape(6, 1),
'no_data_value': -128
}]
tuple_list = [(arrs[0], extents[0]), (arrs[1], extents[1]),
(arrs[2], extents[2])]
ew = BaseTestClass.geopysc._jvm.geopyspark.geotrellis.tests.schemas.KeyValueRecordWrapper
tup = ew.testOut(BaseTestClass.geopysc.sc)
java_rdd = tup._1()
ser = AvroSerializer(tup._2(), decoder, encoder)
rdd = RDD(java_rdd, BaseTestClass.geopysc.pysc, AutoBatchedSerializer(ser))
collected = rdd.collect()
@pytest.fixture(autouse=True)
def tearDown(self):
yield
BaseTestClass.geopysc.pysc._gateway.close()
@pytest.mark.skipif(
'TRAVIS' in os.environ,
reason="Encoding using methods in Main cuases issues on Travis")
def test_encoded_kvs(self):
encoded = self.rdd.map(lambda s: encoder(s))
actual_kvs = encoded.collect()
encoded_tiles = [{
'bands': [{
'cells': bytearray([0, 1, 2, 3, 4, 5]),
'rows': 3,
'cols': 2,
'noDataValue': -128
}]
}, {
'bands': [{
'cells': bytearray([0, 1, 2, 3, 4, 5]),
'rows': 2,
'cols': 3,
'noDataValue': -128
}]
}, {
'bands': [{
'cells': bytearray([0, 1, 2, 3, 4, 5]),
'rows': 6,
'cols': 1,
'noDataValue': -128
}]
}]
encoded_tuples = [{
'_1': encoded_tiles[0],
'_2': self.extents[0]
}, {
'_1': encoded_tiles[1],
'_2': self.extents[1]
}, {
'_1': encoded_tiles[2],
'_2': self.extents[2]
}]
expected_kvs = [
{
'pairs': encoded_tuples
},
{
'pairs': encoded_tuples
},
]
for actual, expected in zip(actual_kvs, expected_kvs):
actual_pairs = actual['pairs']
expected_pairs = expected['pairs']
for a, e in zip(actual_pairs, expected_pairs):
self.assertDictEqual(a['_2'], e['_2'])
def test_decoded_kvs(self):
expected_kvs = [self.tuple_list, self.tuple_list]
for actual_tuples, expected_tuples in zip(self.collected,
expected_kvs):
for actual, expected in zip(actual_tuples, expected_tuples):
(actual_tile, actual_extent) = actual
(expected_tile, expected_extent) = expected
self.assertDictEqual(actual_extent, expected_extent)
def create_value_serializer(self, schema, value_type):
decoder = self.avroregistry._get_decoder(value_type)
encoder = self.avroregistry._get_encoder(value_type)
return AvroSerializer(schema, decoder, encoder)
class TupleSchemaTest(BaseTestClass):
extents = [{
'xmin': 0,
'ymin': 0,
'xmax': 1,
'ymax': 1
}, {
'xmin': 1,
'ymin': 2,
'xmax': 3,
'ymax': 4
}, {
'xmin': 5,
'ymin': 6,
'xmax': 7,
'ymax': 8
}]
arrs = [{
'data': np.array([0, 1, 2, 3, 4, 5]).reshape(1, 3, 2),
'no_data_value': -2147483648
}, {
'data': np.array([0, 1, 2, 3, 4, 5]).reshape(1, 2, 3),
'no_data_value': -2147483648
}, {
'data': np.array([0, 1, 2, 3, 4, 5]).reshape(1, 6, 1),
'no_data_value': -2147483648
}]
sc = BaseTestClass.geopysc.pysc._jsc.sc()
ew = BaseTestClass.geopysc.pysc._jvm.geopyspark.geotrellis.tests.schemas.TupleWrapper
tup = ew.testOut(sc)
java_rdd = tup._1()
ser = AvroSerializer(tup._2(), decoder, encoder)
rdd = RDD(java_rdd, BaseTestClass.geopysc.pysc, AutoBatchedSerializer(ser))
collected = rdd.collect()
@pytest.mark.skipif(
'TRAVIS' in os.environ,
reason="Encoding using methods in Main causes issues on Travis")
def test_encoded_tuples(self):
s = self.rdd._jrdd_deserializer.serializer
encoded = self.rdd.map(lambda x: encoder(x))
actual_encoded = encoded.collect()
expected_encoded = [{
'_1': AvroRegistry.tile_encoder(self.arrs[0]),
'_2': self.extents[0]
}, {
'_1': AvroRegistry.tile_encoder(self.arrs[1]),
'_2': self.extents[1]
}, {
'_1': AvroRegistry.tile_encoder(self.arrs[2]),
'_2': self.extents[2]
}]
for actual, expected in zip(actual_encoded, expected_encoded):
self.assertDictEqual(actual, expected)
def test_decoded_tuples(self):
expected_tuples = [(self.arrs[0], self.extents[0]),
(self.arrs[1], self.extents[1]),
(self.arrs[2], self.extents[2])]
for actual, expected in zip(self.collected, expected_tuples):
(actual_tile, actual_extent) = actual
(expected_tile, expected_extent) = expected
self.assertTrue(
(actual_tile['data'] == expected_tile['data']).all())
self.assertDictEqual(actual_extent, expected_extent)