def test_raster_warp(self): # Create in memory raster source = GDALRaster({ 'datatype': 1, 'driver': 'MEM', 'name': 'sourceraster', 'width': 4, 'height': 4, 'nr_of_bands': 1, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(16), 'nodata_value': 255, }], }) # Test altering the scale, width, and height of a raster data = { 'scale': [200, -200], 'width': 2, 'height': 2, } target = source.warp(data) self.assertEqual(target.width, data['width']) self.assertEqual(target.height, data['height']) self.assertEqual(target.scale, data['scale']) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertEqual(target.name, 'sourceraster_copy.MEM') result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [5, 7, 13, 15]) # Test altering the name and datatype (to float) data = { 'name': '/path/to/targetraster.tif', 'datatype': 6, } target = source.warp(data) self.assertEqual(target.bands[0].datatype(), 6) self.assertEqual(target.name, '/path/to/targetraster.tif') self.assertEqual(target.driver.name, 'MEM') result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [ 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0 ])
def test_raster_warp(self): # Create in memory raster source = GDALRaster({ 'datatype': 1, 'driver': 'MEM', 'name': 'sourceraster', 'width': 4, 'height': 4, 'nr_of_bands': 1, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(16), 'nodata_value': 255, }], }) # Test altering the scale, width, and height of a raster data = { 'scale': [200, -200], 'width': 2, 'height': 2, } target = source.warp(data) self.assertEqual(target.width, data['width']) self.assertEqual(target.height, data['height']) self.assertEqual(target.scale, data['scale']) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertEqual(target.name, 'sourceraster_copy.MEM') result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [5, 7, 13, 15]) # Test altering the name and datatype (to float) data = { 'name': '/path/to/targetraster.tif', 'datatype': 6, } target = source.warp(data) self.assertEqual(target.bands[0].datatype(), 6) self.assertEqual(target.name, '/path/to/targetraster.tif') self.assertEqual(target.driver.name, 'MEM') result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual( result, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0] )
def test_raster_warp_nodata_zone(self): # Create in memory raster. source = GDALRaster({ 'datatype': 1, 'driver': 'MEM', 'width': 4, 'height': 4, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(16), 'nodata_value': 23, }], }) # Warp raster onto a location that does not cover any pixels of the original. result = source.warp({'origin': (200000, 200000)}).bands[0].data() if numpy: result = result.flatten().tolist() # The result is an empty raster filled with the correct nodata value. self.assertEqual(result, [23] * 16)
def test_raster_warp_nodata_zone(self): # Create in memory raster. source = GDALRaster({ "datatype": 1, "driver": "MEM", "width": 4, "height": 4, "srid": 3086, "origin": (500000, 400000), "scale": (100, -100), "skew": (0, 0), "bands": [{ "data": range(16), "nodata_value": 23, }], }) # Warp raster onto a location that does not cover any pixels of the original. result = source.warp({"origin": (200000, 200000)}).bands[0].data() if numpy: result = result.flatten().tolist() # The result is an empty raster filled with the correct nodata value. self.assertEqual(result, [23] * 16)
def test_raster_warp(self): # Create in memory raster source = GDALRaster( { "datatype": 1, "driver": "MEM", "name": "sourceraster", "width": 4, "height": 4, "nr_of_bands": 1, "srid": 3086, "origin": (500000, 400000), "scale": (100, -100), "skew": (0, 0), "bands": [{"data": range(16), "nodata_value": 255}], } ) # Test altering the scale, width, and height of a raster data = {"scale": [200, -200], "width": 2, "height": 2} target = source.warp(data) self.assertEqual(target.width, data["width"]) self.assertEqual(target.height, data["height"]) self.assertEqual(target.scale, data["scale"]) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertEqual(target.name, "sourceraster_copy.MEM") result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [5, 7, 13, 15]) # Test altering the name and datatype (to float) data = {"name": "/path/to/targetraster.tif", "datatype": 6} target = source.warp(data) self.assertEqual(target.bands[0].datatype(), 6) self.assertEqual(target.name, "/path/to/targetraster.tif") self.assertEqual(target.driver.name, "MEM") result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0])
class GDALRasterTests(SimpleTestCase): """ Test a GDALRaster instance created from a file (GeoTiff). """ def setUp(self): self.rs_path = os.path.join(os.path.dirname(__file__), '../data/rasters/raster.tif') self.rs = GDALRaster(self.rs_path) def test_rs_name_repr(self): self.assertEqual(self.rs_path, self.rs.name) self.assertRegex(repr(self.rs), r"<Raster object at 0x\w+>") def test_rs_driver(self): self.assertEqual(self.rs.driver.name, 'GTiff') def test_rs_size(self): self.assertEqual(self.rs.width, 163) self.assertEqual(self.rs.height, 174) def test_rs_srs(self): self.assertEqual(self.rs.srs.srid, 3086) self.assertEqual(self.rs.srs.units, (1.0, 'metre')) def test_rs_srid(self): rast = GDALRaster({ 'width': 16, 'height': 16, 'srid': 4326, }) self.assertEqual(rast.srid, 4326) rast.srid = 3086 self.assertEqual(rast.srid, 3086) def test_geotransform_and_friends(self): # Assert correct values for file based raster self.assertEqual( self.rs.geotransform, [511700.4680706557, 100.0, 0.0, 435103.3771231986, 0.0, -100.0]) self.assertEqual(self.rs.origin, [511700.4680706557, 435103.3771231986]) self.assertEqual(self.rs.origin.x, 511700.4680706557) self.assertEqual(self.rs.origin.y, 435103.3771231986) self.assertEqual(self.rs.scale, [100.0, -100.0]) self.assertEqual(self.rs.scale.x, 100.0) self.assertEqual(self.rs.scale.y, -100.0) self.assertEqual(self.rs.skew, [0, 0]) self.assertEqual(self.rs.skew.x, 0) self.assertEqual(self.rs.skew.y, 0) # Create in-memory rasters and change gtvalues rsmem = GDALRaster(JSON_RASTER) # geotransform accepts both floats and ints rsmem.geotransform = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] self.assertEqual(rsmem.geotransform, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]) rsmem.geotransform = range(6) self.assertEqual(rsmem.geotransform, [float(x) for x in range(6)]) self.assertEqual(rsmem.origin, [0, 3]) self.assertEqual(rsmem.origin.x, 0) self.assertEqual(rsmem.origin.y, 3) self.assertEqual(rsmem.scale, [1, 5]) self.assertEqual(rsmem.scale.x, 1) self.assertEqual(rsmem.scale.y, 5) self.assertEqual(rsmem.skew, [2, 4]) self.assertEqual(rsmem.skew.x, 2) self.assertEqual(rsmem.skew.y, 4) self.assertEqual(rsmem.width, 5) self.assertEqual(rsmem.height, 5) def test_geotransform_bad_inputs(self): rsmem = GDALRaster(JSON_RASTER) error_geotransforms = [ [1, 2], [1, 2, 3, 4, 5, 'foo'], [1, 2, 3, 4, 5, 6, 'foo'], ] msg = 'Geotransform must consist of 6 numeric values.' for geotransform in error_geotransforms: with self.subTest(i=geotransform), self.assertRaisesMessage( ValueError, msg): rsmem.geotransform = geotransform def test_rs_extent(self): self.assertEqual(self.rs.extent, (511700.4680706557, 417703.3771231986, 528000.4680706557, 435103.3771231986)) def test_rs_bands(self): self.assertEqual(len(self.rs.bands), 1) self.assertIsInstance(self.rs.bands[0], GDALBand) def test_memory_based_raster_creation(self): # Create uint8 raster with full pixel data range (0-255) rast = GDALRaster({ 'datatype': 1, 'width': 16, 'height': 16, 'srid': 4326, 'bands': [{ 'data': range(256), 'nodata_value': 255, }], }) # Get array from raster result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Assert data is same as original input self.assertEqual(result, list(range(256))) def test_file_based_raster_creation(self): # Prepare tempfile rstfile = tempfile.NamedTemporaryFile(suffix='.tif') # Create file-based raster from scratch GDALRaster({ 'datatype': self.rs.bands[0].datatype(), 'driver': 'tif', 'name': rstfile.name, 'width': 163, 'height': 174, 'nr_of_bands': 1, 'srid': self.rs.srs.wkt, 'origin': (self.rs.origin.x, self.rs.origin.y), 'scale': (self.rs.scale.x, self.rs.scale.y), 'skew': (self.rs.skew.x, self.rs.skew.y), 'bands': [{ 'data': self.rs.bands[0].data(), 'nodata_value': self.rs.bands[0].nodata_value, }], }) # Reload newly created raster from file restored_raster = GDALRaster(rstfile.name) self.assertEqual(restored_raster.srs.wkt, self.rs.srs.wkt) self.assertEqual(restored_raster.geotransform, self.rs.geotransform) if numpy: numpy.testing.assert_equal(restored_raster.bands[0].data(), self.rs.bands[0].data()) else: self.assertEqual(restored_raster.bands[0].data(), self.rs.bands[0].data()) def test_vsi_raster_creation(self): # Open a raster as a file object. with open(self.rs_path, 'rb') as dat: # Instantiate a raster from the file binary buffer. vsimem = GDALRaster(dat.read()) # The data of the in-memory file is equal to the source file. result = vsimem.bands[0].data() target = self.rs.bands[0].data() if numpy: result = result.flatten().tolist() target = target.flatten().tolist() self.assertEqual(result, target) def test_vsi_raster_deletion(self): path = '/vsimem/raster.tif' # Create a vsi-based raster from scratch. vsimem = GDALRaster({ 'name': path, 'driver': 'tif', 'width': 4, 'height': 4, 'srid': 4326, 'bands': [{ 'data': range(16), }], }) # The virtual file exists. rst = GDALRaster(path) self.assertEqual(rst.width, 4) # Delete GDALRaster. del vsimem del rst # The virtual file has been removed. msg = 'Could not open the datasource at "/vsimem/raster.tif"' with self.assertRaisesMessage(GDALException, msg): GDALRaster(path) def test_vsi_invalid_buffer_error(self): msg = 'Failed creating VSI raster from the input buffer.' with self.assertRaisesMessage(GDALException, msg): GDALRaster(b'not-a-raster-buffer') def test_vsi_buffer_property(self): # Create a vsi-based raster from scratch. rast = GDALRaster({ 'name': '/vsimem/raster.tif', 'driver': 'tif', 'width': 4, 'height': 4, 'srid': 4326, 'bands': [{ 'data': range(16), }], }) # Do a round trip from raster to buffer to raster. result = GDALRaster(rast.vsi_buffer).bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to nodata value except on input block of ones. self.assertEqual(result, list(range(16))) # The vsi buffer is None for rasters that are not vsi based. self.assertIsNone(self.rs.vsi_buffer) def test_offset_size_and_shape_on_raster_creation(self): rast = GDALRaster({ 'datatype': 1, 'width': 4, 'height': 4, 'srid': 4326, 'bands': [{ 'data': (1, ), 'offset': (1, 1), 'size': (2, 2), 'shape': (1, 1), 'nodata_value': 2, }], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to nodata value except on input block of ones. self.assertEqual(result, [2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2]) def test_set_nodata_value_on_raster_creation(self): # Create raster filled with nodata values. rast = GDALRaster({ 'datatype': 1, 'width': 2, 'height': 2, 'srid': 4326, 'bands': [{ 'nodata_value': 23 }], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # All band data is equal to nodata value. self.assertEqual(result, [23] * 4) def test_set_nodata_none_on_raster_creation(self): if GDAL_VERSION < (2, 1): self.skipTest("GDAL >= 2.1 is required for this test.") # Create raster without data and without nodata value. rast = GDALRaster({ 'datatype': 1, 'width': 2, 'height': 2, 'srid': 4326, 'bands': [{ 'nodata_value': None }], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to zero because no nodata value has been specified. self.assertEqual(result, [0] * 4) def test_raster_metadata_property(self): data = self.rs.metadata self.assertEqual(data['DEFAULT'], {'AREA_OR_POINT': 'Area'}) self.assertEqual(data['IMAGE_STRUCTURE'], {'INTERLEAVE': 'BAND'}) # Create file-based raster from scratch source = GDALRaster({ 'datatype': 1, 'width': 2, 'height': 2, 'srid': 4326, 'bands': [{ 'data': range(4), 'nodata_value': 99 }], }) # Set metadata on raster and on a band. metadata = { 'DEFAULT': { 'OWNER': 'Django', 'VERSION': '1.0', 'AREA_OR_POINT': 'Point' }, } source.metadata = metadata source.bands[0].metadata = metadata self.assertEqual(source.metadata['DEFAULT'], metadata['DEFAULT']) self.assertEqual(source.bands[0].metadata['DEFAULT'], metadata['DEFAULT']) # Update metadata on raster. metadata = { 'DEFAULT': { 'VERSION': '2.0' }, } source.metadata = metadata self.assertEqual(source.metadata['DEFAULT']['VERSION'], '2.0') # Remove metadata on raster. metadata = { 'DEFAULT': { 'OWNER': None }, } source.metadata = metadata self.assertNotIn('OWNER', source.metadata['DEFAULT']) def test_raster_info_accessor(self): if GDAL_VERSION < (2, 1): msg = 'GDAL ≥ 2.1 is required for using the info property.' with self.assertRaisesMessage(ValueError, msg): self.rs.info return gdalinfo = """ Driver: GTiff/GeoTIFF Files: {} Size is 163, 174 Coordinate System is: PROJCS["NAD83 / Florida GDL Albers", GEOGCS["NAD83", DATUM["North_American_Datum_1983", SPHEROID["GRS 1980",6378137,298.257222101, AUTHORITY["EPSG","7019"]], TOWGS84[0,0,0,0,0,0,0], AUTHORITY["EPSG","6269"]], PRIMEM["Greenwich",0, AUTHORITY["EPSG","8901"]], UNIT["degree",0.0174532925199433, AUTHORITY["EPSG","9122"]], AUTHORITY["EPSG","4269"]], PROJECTION["Albers_Conic_Equal_Area"], PARAMETER["standard_parallel_1",24], PARAMETER["standard_parallel_2",31.5], PARAMETER["latitude_of_center",24], PARAMETER["longitude_of_center",-84], PARAMETER["false_easting",400000], PARAMETER["false_northing",0], UNIT["metre",1, AUTHORITY["EPSG","9001"]], AXIS["X",EAST], AXIS["Y",NORTH], AUTHORITY["EPSG","3086"]] Origin = (511700.468070655711927,435103.377123198588379) Pixel Size = (100.000000000000000,-100.000000000000000) Metadata: AREA_OR_POINT=Area Image Structure Metadata: INTERLEAVE=BAND Corner Coordinates: Upper Left ( 511700.468, 435103.377) ( 82d51'46.16"W, 27d55' 1.53"N) Lower Left ( 511700.468, 417703.377) ( 82d51'52.04"W, 27d45'37.50"N) Upper Right ( 528000.468, 435103.377) ( 82d41'48.81"W, 27d54'56.30"N) Lower Right ( 528000.468, 417703.377) ( 82d41'55.54"W, 27d45'32.28"N) Center ( 519850.468, 426403.377) ( 82d46'50.64"W, 27d50'16.99"N) Band 1 Block=163x50 Type=Byte, ColorInterp=Gray NoData Value=15 """.format(self.rs_path) # Data info_dyn = [ line.strip() for line in self.rs.info.split('\n') if line.strip() != '' ] info_ref = [ line.strip() for line in gdalinfo.split('\n') if line.strip() != '' ] self.assertEqual(info_dyn, info_ref) def test_compressed_file_based_raster_creation(self): rstfile = tempfile.NamedTemporaryFile(suffix='.tif') # Make a compressed copy of an existing raster. compressed = self.rs.warp({ 'papsz_options': { 'compress': 'packbits' }, 'name': rstfile.name }) # Check physically if compression worked. self.assertLess(os.path.getsize(compressed.name), os.path.getsize(self.rs.name)) # Create file-based raster with options from scratch. compressed = GDALRaster({ 'datatype': 1, 'driver': 'tif', 'name': rstfile.name, 'width': 40, 'height': 40, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(40 ^ 2), 'nodata_value': 255, }], 'papsz_options': { 'compress': 'packbits', 'pixeltype': 'signedbyte', 'blockxsize': 23, 'blockysize': 23, } }) # Check if options used on creation are stored in metadata. # Reopening the raster ensures that all metadata has been written # to the file. compressed = GDALRaster(compressed.name) self.assertEqual( compressed.metadata['IMAGE_STRUCTURE']['COMPRESSION'], 'PACKBITS', ) self.assertEqual( compressed.bands[0].metadata['IMAGE_STRUCTURE']['PIXELTYPE'], 'SIGNEDBYTE') if GDAL_VERSION >= (2, 1): self.assertIn('Block=40x23', compressed.info) def test_raster_warp(self): # Create in memory raster source = GDALRaster({ 'datatype': 1, 'driver': 'MEM', 'name': 'sourceraster', 'width': 4, 'height': 4, 'nr_of_bands': 1, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(16), 'nodata_value': 255, }], }) # Test altering the scale, width, and height of a raster data = { 'scale': [200, -200], 'width': 2, 'height': 2, } target = source.warp(data) self.assertEqual(target.width, data['width']) self.assertEqual(target.height, data['height']) self.assertEqual(target.scale, data['scale']) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertEqual(target.name, 'sourceraster_copy.MEM') result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [5, 7, 13, 15]) # Test altering the name and datatype (to float) data = { 'name': '/path/to/targetraster.tif', 'datatype': 6, } target = source.warp(data) self.assertEqual(target.bands[0].datatype(), 6) self.assertEqual(target.name, '/path/to/targetraster.tif') self.assertEqual(target.driver.name, 'MEM') result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [ 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0 ]) def test_raster_warp_nodata_zone(self): # Create in memory raster. source = GDALRaster({ 'datatype': 1, 'driver': 'MEM', 'width': 4, 'height': 4, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(16), 'nodata_value': 23, }], }) # Warp raster onto a location that does not cover any pixels of the original. result = source.warp({'origin': (200000, 200000)}).bands[0].data() if numpy: result = result.flatten().tolist() # The result is an empty raster filled with the correct nodata value. self.assertEqual(result, [23] * 16) def test_raster_transform(self): # Prepare tempfile and nodata value rstfile = tempfile.NamedTemporaryFile(suffix='.tif') ndv = 99 # Create in file based raster source = GDALRaster({ 'datatype': 1, 'driver': 'tif', 'name': rstfile.name, 'width': 5, 'height': 5, 'nr_of_bands': 1, 'srid': 4326, 'origin': (-5, 5), 'scale': (2, -2), 'skew': (0, 0), 'bands': [{ 'data': range(25), 'nodata_value': ndv, }], }) # Transform raster into srid 4326. target = source.transform(3086) # Reload data from disk target = GDALRaster(target.name) self.assertEqual(target.srs.srid, 3086) self.assertEqual(target.width, 7) self.assertEqual(target.height, 7) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertAlmostEqual(target.origin[0], 9124842.791079799, 3) self.assertAlmostEqual(target.origin[1], 1589911.6476407414, 3) self.assertAlmostEqual(target.scale[0], 223824.82664250192, 3) self.assertAlmostEqual(target.scale[1], -223824.82664250192, 3) self.assertEqual(target.skew, [0, 0]) result = target.bands[0].data() if numpy: result = result.flatten().tolist() # The reprojection of a raster that spans over a large area # skews the data matrix and might introduce nodata values. self.assertEqual(result, [ ndv, ndv, ndv, ndv, 4, ndv, ndv, ndv, ndv, 2, 3, 9, ndv, ndv, ndv, 1, 2, 8, 13, 19, ndv, 0, 6, 6, 12, 18, 18, 24, ndv, 10, 11, 16, 22, 23, ndv, ndv, ndv, 15, 21, 22, ndv, ndv, ndv, ndv, 20, ndv, ndv, ndv, ndv, ])
class RasterLayerParser(object): """ Class to parse raster layers. """ def __init__(self, rasterlayer_id): self.rasterlayer = RasterLayer.objects.get(id=rasterlayer_id) # Set raster tilesize self.tilesize = int( getattr(settings, 'RASTER_TILESIZE', WEB_MERCATOR_TILESIZE)) self.batch_step_size = int( getattr(settings, 'RASTER_BATCH_STEP_SIZE', BATCH_STEP_SIZE)) self.s3_endpoint_url = getattr(settings, 'RASTER_S3_ENDPOINT_URL', None) def log(self, msg, status=None, zoom=None): """ Write a message to the parse log of the rasterlayer instance and update the parse status object. """ parsestatus = self.rasterlayer.parsestatus parsestatus.refresh_from_db() if status is not None: parsestatus.status = status if zoom is not None and zoom not in parsestatus.tile_levels: parsestatus.tile_levels.append(zoom) parsestatus.tile_levels.sort() # Prepare datetime stamp for log now = '[{0}] '.format( datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) if parsestatus.log: now = '\n' + now parsestatus.log += now + msg parsestatus.save() def open_raster_file(self): """ Get raster source file to extract tiles from. This makes a local copy of rasterfile, unzips the raster and reprojects it into web mercator if necessary. The reprojected raster is stored for reuse such that reprojection does only happen once. The local copy of the raster is needed if files are stored on remote storages. """ reproj, created = RasterLayerReprojected.objects.get_or_create( rasterlayer=self.rasterlayer) # Check if the raster has already been reprojected has_reprojected = reproj.rasterfile.name not in (None, '') # Create workdir raster_workdir = getattr(settings, 'RASTER_WORKDIR', None) self.tmpdir = tempfile.mkdtemp(dir=raster_workdir) # Choose source for raster data, use the reprojected version if it exists. if self.rasterlayer.source_url and not has_reprojected: url = self.rasterlayer.source_url if url.lower().startswith('http') or url.startswith('file'): url_path = urlparse(self.rasterlayer.source_url).path filename = url_path.split('/')[-1] filepath = os.path.join(self.tmpdir, filename) urlretrieve(self.rasterlayer.source_url, filepath) elif url.startswith('s3'): # Get the bucket name and file key, assuming the following url # strucure: s3://BUCKET_NAME/BUCKET_KEY bucket_name = url.split('s3://')[1].split('/')[0] bucket_key = '/'.join(url.split('s3://')[1].split('/')[1:]) # Assume the file name is the last piece of the key. filename = bucket_key.split('/')[-1] filepath = os.path.join(self.tmpdir, filename) # Get file from s3. s3 = boto3.resource('s3', endpoint_url=self.s3_endpoint_url) bucket = s3.Bucket(bucket_name) bucket.download_file(bucket_key, filepath, ExtraArgs={'RequestPayer': 'requester'}) else: raise RasterException( 'Only http(s) and s3 urls are supported.') else: if has_reprojected: rasterfile_source = reproj.rasterfile else: rasterfile_source = self.rasterlayer.rasterfile if not rasterfile_source.name: raise RasterException( 'No data source found. Provide a rasterfile or a source url.' ) # Copy raster file source to local folder filepath = os.path.join(self.tmpdir, os.path.basename(rasterfile_source.name)) rasterfile = open(filepath, 'wb') for chunk in rasterfile_source.chunks(): rasterfile.write(chunk) rasterfile.close() # If the raster file is compressed, decompress it, otherwise try to # open the source file directly. if os.path.splitext(filepath)[1].lower() == '.zip': # Open and extract zipfile zf = zipfile.ZipFile(filepath) zf.extractall(self.tmpdir) # Remove zipfile os.remove(filepath) # Get filelist from directory matches = [] for root, dirnames, filenames in os.walk(self.tmpdir): for filename in fnmatch.filter(filenames, '*.*'): matches.append(os.path.join(root, filename)) # Open the first raster file found in the matched files. self.dataset = None for match in matches: try: self.dataset = GDALRaster(match) break except GDALException: pass # Raise exception if no file could be opened by gdal. if not self.dataset: raise RasterException('Could not open rasterfile.') else: self.dataset = GDALRaster(filepath) # Override srid if provided if self.rasterlayer.srid: try: self.dataset = GDALRaster(self.dataset.name, write=True) except GDALException: raise RasterException( 'Could not override srid because the driver for this ' 'type of raster does not support write mode.') self.dataset.srs = self.rasterlayer.srid def reproject_rasterfile(self): """ Reproject the rasterfile into web mercator. """ # Return if reprojected rasterfile already exists. if hasattr(self.rasterlayer, 'reprojected' ) and self.rasterlayer.reprojected.rasterfile.name: return # Return if the raster already has the right projection # and nodata value is acceptable. if self.dataset.srs.srid == WEB_MERCATOR_SRID: # SRID was not manually specified. if self.rasterlayer.nodata in ('', None): return # All bands from dataset already have the same nodata value as the # one that was manually specified. if all([ self.rasterlayer.nodata == band.nodata_value for band in self.dataset.bands ]): return else: # Log projection change if original raster is not in web mercator. self.log( 'Transforming raster to SRID {0}'.format(WEB_MERCATOR_SRID), status=self.rasterlayer.parsestatus.REPROJECTING_RASTER, ) # Reproject the dataset. self.dataset = self.dataset.transform( WEB_MERCATOR_SRID, driver=INTERMEDIATE_RASTER_FORMAT, ) # Manually override nodata value if neccessary if self.rasterlayer.nodata not in ('', None): self.log( 'Setting no data values to {0}.'.format( self.rasterlayer.nodata), status=self.rasterlayer.parsestatus.REPROJECTING_RASTER, ) for band in self.dataset.bands: band.nodata_value = float(self.rasterlayer.nodata) # Compress reprojected raster file and store it if self.rasterlayer.store_reprojected: dest = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.zip') dest_zip = zipfile.ZipFile(dest.name, 'w', allowZip64=True) dest_zip.write( filename=self.dataset.name, arcname=os.path.basename(self.dataset.name), compress_type=zipfile.ZIP_DEFLATED, ) dest_zip.close() # Store zip file in reprojected raster model self.rasterlayer.reprojected.rasterfile = File( open(dest_zip.filename, 'rb'), name=os.path.basename(dest_zip.filename)) self.rasterlayer.reprojected.save() self.log('Finished transforming raster.') def create_initial_histogram_buckets(self): """ Gets the empty histogram arrays for statistics collection. """ self.hist_values = [] self.hist_bins = [] for i, band in enumerate(self.dataset.bands): bandmeta = RasterLayerBandMetadata.objects.filter( rasterlayer=self.rasterlayer, band=i).first() self.hist_values.append(numpy.array(bandmeta.hist_values)) self.hist_bins.append(numpy.array(bandmeta.hist_bins)) def extract_metadata(self): """ Extract and store metadata for the raster and its bands. """ self.log('Extracting metadata from raster.') # Try to compute max zoom try: max_zoom = self.compute_max_zoom() except GDALException: raise RasterException( 'Failed to compute max zoom. Check the SRID of the raster.') # Extract global raster metadata meta = self.rasterlayer.metadata meta.uperleftx = self.dataset.origin.x meta.uperlefty = self.dataset.origin.y meta.width = self.dataset.width meta.height = self.dataset.height meta.scalex = self.dataset.scale.x meta.scaley = self.dataset.scale.y meta.skewx = self.dataset.skew.x meta.skewy = self.dataset.skew.y meta.numbands = len(self.dataset.bands) meta.srs_wkt = self.dataset.srs.wkt meta.srid = self.dataset.srs.srid meta.max_zoom = max_zoom meta.save() # Extract band metadata for i, band in enumerate(self.dataset.bands): bandmeta = RasterLayerBandMetadata.objects.filter( rasterlayer=self.rasterlayer, band=i).first() if not bandmeta: bandmeta = RasterLayerBandMetadata( rasterlayer=self.rasterlayer, band=i) bandmeta.nodata_value = band.nodata_value bandmeta.min = band.min bandmeta.max = band.max # Depending on Django version, the band statistics include std and mean. if hasattr(band, 'std'): bandmeta.std = band.std if hasattr(band, 'mean'): bandmeta.mean = band.mean bandmeta.save() self.log('Finished extracting metadata from raster.') def create_tiles(self, zoom_levels): """ Create tiles for input zoom levels, either a list or an integer. """ if isinstance(zoom_levels, int): self.populate_tile_level(zoom_levels) else: for zoom in zoom_levels: self.populate_tile_level(zoom) def populate_tile_level(self, zoom): """ Create tiles for this raster at the given zoomlevel. This routine first snaps the raster to the grid of the zoomlevel, then creates the tiles from the snapped raster. """ # Abort if zoom level is above resolution of the raster layer if zoom > self.max_zoom: return elif zoom == self.max_zoom: self.create_initial_histogram_buckets() # Compute the tile x-y-z index range for the rasterlayer for this zoomlevel bbox = self.dataset.extent quadrants = utils.quadrants(bbox, zoom) self.log('Creating {0} tiles in {1} quadrants at zoom {2}.'.format( self.nr_of_tiles(zoom), len(quadrants), zoom)) # Process quadrants in parallell for indexrange in quadrants: self.process_quadrant(indexrange, zoom) # Store histogram data if zoom == self.max_zoom: bandmetas = RasterLayerBandMetadata.objects.filter( rasterlayer=self.rasterlayer) for bandmeta in bandmetas: bandmeta.hist_values = self.hist_values[bandmeta.band].tolist() bandmeta.save() self.log('Finished parsing at zoom level {0}.'.format(zoom), zoom=zoom) _quadrant_count = 0 def process_quadrant(self, indexrange, zoom): """ Create raster tiles for a quadrant of tiles defined by a x-y-z index range and a zoom level. """ # TODO Use a standalone celery task for this method in order to # gain speedup from parallelism. self._quadrant_count += 1 self.log( 'Starting tile creation for quadrant {0} at zoom level {1}'.format( self._quadrant_count, zoom), status=self.rasterlayer.parsestatus.CREATING_TILES) # Compute scale of tiles for this zoomlevel tilescale = utils.tile_scale(zoom) # Compute quadrant bounds and create destination file bounds = utils.tile_bounds(indexrange[0], indexrange[1], zoom) dest_file = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.tif') # Snap dataset to the quadrant snapped_dataset = self.dataset.warp({ 'name': dest_file.name, 'origin': [bounds[0], bounds[3]], 'scale': [tilescale, -tilescale], 'width': (indexrange[2] - indexrange[0] + 1) * self.tilesize, 'height': (indexrange[3] - indexrange[1] + 1) * self.tilesize, }) # Create all tiles in this quadrant in batches batch = [] for tilex in range(indexrange[0], indexrange[2] + 1): for tiley in range(indexrange[1], indexrange[3] + 1): # Calculate raster tile origin bounds = utils.tile_bounds(tilex, tiley, zoom) # Construct band data arrays pixeloffset = ((tilex - indexrange[0]) * self.tilesize, (tiley - indexrange[1]) * self.tilesize) band_data = [{ 'data': band.data(offset=pixeloffset, size=(self.tilesize, self.tilesize)), 'nodata_value': band.nodata_value } for band in snapped_dataset.bands] # Ignore tile if its only nodata. if all([ numpy.all(dat['data'] == dat['nodata_value']) for dat in band_data ]): continue # Add tile data to histogram if zoom == self.max_zoom: self.push_histogram(band_data) # Warp source raster into this tile (in memory) dest = GDALRaster({ 'width': self.tilesize, 'height': self.tilesize, 'origin': [bounds[0], bounds[3]], 'scale': [tilescale, -tilescale], 'srid': WEB_MERCATOR_SRID, 'datatype': snapped_dataset.bands[0].datatype(), 'bands': band_data, }) # Store tile in batch array batch.append( RasterTile(rast=dest, rasterlayer_id=self.rasterlayer.id, tilex=tilex, tiley=tiley, tilez=zoom)) # Commit batch to database and reset it if len(batch) == self.batch_step_size: RasterTile.objects.bulk_create(batch) batch = [] # Commit remaining objects if len(batch): RasterTile.objects.bulk_create(batch) def push_histogram(self, data): """ Add data to band level histogram. """ # Loop through bands of this tile for i, dat in enumerate(data): # Create histogram for new data with the same bins new_hist = numpy.histogram(dat['data'], bins=self.hist_bins[i]) # Add counts of this tile to band metadata histogram self.hist_values[i] += new_hist[0] def drop_all_tiles(self): """ Delete all existing tiles for this parser's rasterlayer. """ self.log('Clearing all existing tiles.') self.rasterlayer.rastertile_set.all().delete() self.log('Finished clearing existing tiles.') def send_success_signal(self): """ Send parser end signal for other dependencies to be handling new tiles. """ self.log('Successfully finished parsing raster', status=self.rasterlayer.parsestatus.FINISHED) rasterlayers_parser_ended.send(sender=self.rasterlayer.__class__, instance=self.rasterlayer) def compute_max_zoom(self): """ Set max zoom property based on rasterlayer metadata. """ # Return manual override value if provided if self.rasterlayer.max_zoom is not None: return self.rasterlayer.max_zoom if self.dataset.srs.srid == WEB_MERCATOR_SRID: # For rasters in web mercator, use the scale directly scale = abs(self.dataset.scale.x) else: # Create a line from the center of the raster to a point that is # one pixel width away from the center. xcenter = self.dataset.extent[0] + (self.dataset.extent[2] - self.dataset.extent[0]) / 2 ycenter = self.dataset.extent[1] + (self.dataset.extent[3] - self.dataset.extent[1]) / 2 linestring = 'LINESTRING({} {}, {} {})'.format( xcenter, ycenter, xcenter + self.dataset.scale.x, ycenter) line = OGRGeometry(linestring, srs=self.dataset.srs) # Tansform the line into web mercator. line.transform(WEB_MERCATOR_SRID) # Use the lenght of the transformed line as scale. scale = line.geos.length return utils.closest_zoomlevel(scale) @property def max_zoom(self): # Return manual override value if provided if self.rasterlayer.max_zoom is not None: return self.rasterlayer.max_zoom # Get max zoom from metadata if not hasattr(self.rasterlayer, 'metadata'): raise RasterException('Could not determine max zoom level.') max_zoom = self.rasterlayer.metadata.max_zoom # Reduce max zoom by one if zoomdown flag was disabled if not self.rasterlayer.next_higher: max_zoom -= 1 return max_zoom def nr_of_tiles(self, zoom): """ Compute the number of tiles for the rasterlayer on a given zoom level. """ bbox = self.dataset.extent indexrange = utils.tile_index_range(bbox, zoom) return (indexrange[2] - indexrange[0] + 1) * (indexrange[3] - indexrange[1] + 1)
class GDALRasterTests(SimpleTestCase): """ Test a GDALRaster instance created from a file (GeoTiff). """ def setUp(self): self.rs_path = os.path.join(os.path.dirname(__file__), '../data/rasters/raster.tif') self.rs = GDALRaster(self.rs_path) def test_rs_name_repr(self): self.assertEqual(self.rs_path, self.rs.name) self.assertRegex(repr(self.rs), r"<Raster object at 0x\w+>") def test_rs_driver(self): self.assertEqual(self.rs.driver.name, 'GTiff') def test_rs_size(self): self.assertEqual(self.rs.width, 163) self.assertEqual(self.rs.height, 174) def test_rs_srs(self): self.assertEqual(self.rs.srs.srid, 3086) self.assertEqual(self.rs.srs.units, (1.0, 'metre')) def test_rs_srid(self): rast = GDALRaster({ 'width': 16, 'height': 16, 'srid': 4326, }) self.assertEqual(rast.srid, 4326) rast.srid = 3086 self.assertEqual(rast.srid, 3086) def test_geotransform_and_friends(self): # Assert correct values for file based raster self.assertEqual( self.rs.geotransform, [511700.4680706557, 100.0, 0.0, 435103.3771231986, 0.0, -100.0] ) self.assertEqual(self.rs.origin, [511700.4680706557, 435103.3771231986]) self.assertEqual(self.rs.origin.x, 511700.4680706557) self.assertEqual(self.rs.origin.y, 435103.3771231986) self.assertEqual(self.rs.scale, [100.0, -100.0]) self.assertEqual(self.rs.scale.x, 100.0) self.assertEqual(self.rs.scale.y, -100.0) self.assertEqual(self.rs.skew, [0, 0]) self.assertEqual(self.rs.skew.x, 0) self.assertEqual(self.rs.skew.y, 0) # Create in-memory rasters and change gtvalues rsmem = GDALRaster(JSON_RASTER) # geotransform accepts both floats and ints rsmem.geotransform = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] self.assertEqual(rsmem.geotransform, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]) rsmem.geotransform = range(6) self.assertEqual(rsmem.geotransform, [float(x) for x in range(6)]) self.assertEqual(rsmem.origin, [0, 3]) self.assertEqual(rsmem.origin.x, 0) self.assertEqual(rsmem.origin.y, 3) self.assertEqual(rsmem.scale, [1, 5]) self.assertEqual(rsmem.scale.x, 1) self.assertEqual(rsmem.scale.y, 5) self.assertEqual(rsmem.skew, [2, 4]) self.assertEqual(rsmem.skew.x, 2) self.assertEqual(rsmem.skew.y, 4) self.assertEqual(rsmem.width, 5) self.assertEqual(rsmem.height, 5) def test_geotransform_bad_inputs(self): rsmem = GDALRaster(JSON_RASTER) error_geotransforms = [ [1, 2], [1, 2, 3, 4, 5, 'foo'], [1, 2, 3, 4, 5, 6, 'foo'], ] msg = 'Geotransform must consist of 6 numeric values.' for geotransform in error_geotransforms: with self.subTest(i=geotransform), self.assertRaisesMessage(ValueError, msg): rsmem.geotransform = geotransform def test_rs_extent(self): self.assertEqual( self.rs.extent, (511700.4680706557, 417703.3771231986, 528000.4680706557, 435103.3771231986) ) def test_rs_bands(self): self.assertEqual(len(self.rs.bands), 1) self.assertIsInstance(self.rs.bands[0], GDALBand) def test_memory_based_raster_creation(self): # Create uint8 raster with full pixel data range (0-255) rast = GDALRaster({ 'datatype': 1, 'width': 16, 'height': 16, 'srid': 4326, 'bands': [{ 'data': range(256), 'nodata_value': 255, }], }) # Get array from raster result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Assert data is same as original input self.assertEqual(result, list(range(256))) def test_file_based_raster_creation(self): # Prepare tempfile rstfile = tempfile.NamedTemporaryFile(suffix='.tif') # Create file-based raster from scratch GDALRaster({ 'datatype': self.rs.bands[0].datatype(), 'driver': 'tif', 'name': rstfile.name, 'width': 163, 'height': 174, 'nr_of_bands': 1, 'srid': self.rs.srs.wkt, 'origin': (self.rs.origin.x, self.rs.origin.y), 'scale': (self.rs.scale.x, self.rs.scale.y), 'skew': (self.rs.skew.x, self.rs.skew.y), 'bands': [{ 'data': self.rs.bands[0].data(), 'nodata_value': self.rs.bands[0].nodata_value, }], }) # Reload newly created raster from file restored_raster = GDALRaster(rstfile.name) # Presence of TOWGS84 depend on GDAL/Proj versions. self.assertEqual( restored_raster.srs.wkt.replace('TOWGS84[0,0,0,0,0,0,0],', ''), self.rs.srs.wkt.replace('TOWGS84[0,0,0,0,0,0,0],', '') ) self.assertEqual(restored_raster.geotransform, self.rs.geotransform) if numpy: numpy.testing.assert_equal( restored_raster.bands[0].data(), self.rs.bands[0].data() ) else: self.assertEqual(restored_raster.bands[0].data(), self.rs.bands[0].data()) def test_nonexistent_file(self): msg = 'Unable to read raster source input "nonexistent.tif".' with self.assertRaisesMessage(GDALException, msg): GDALRaster('nonexistent.tif') def test_vsi_raster_creation(self): # Open a raster as a file object. with open(self.rs_path, 'rb') as dat: # Instantiate a raster from the file binary buffer. vsimem = GDALRaster(dat.read()) # The data of the in-memory file is equal to the source file. result = vsimem.bands[0].data() target = self.rs.bands[0].data() if numpy: result = result.flatten().tolist() target = target.flatten().tolist() self.assertEqual(result, target) def test_vsi_raster_deletion(self): path = '/vsimem/raster.tif' # Create a vsi-based raster from scratch. vsimem = GDALRaster({ 'name': path, 'driver': 'tif', 'width': 4, 'height': 4, 'srid': 4326, 'bands': [{ 'data': range(16), }], }) # The virtual file exists. rst = GDALRaster(path) self.assertEqual(rst.width, 4) # Delete GDALRaster. del vsimem del rst # The virtual file has been removed. msg = 'Could not open the datasource at "/vsimem/raster.tif"' with self.assertRaisesMessage(GDALException, msg): GDALRaster(path) def test_vsi_invalid_buffer_error(self): msg = 'Failed creating VSI raster from the input buffer.' with self.assertRaisesMessage(GDALException, msg): GDALRaster(b'not-a-raster-buffer') def test_vsi_buffer_property(self): # Create a vsi-based raster from scratch. rast = GDALRaster({ 'name': '/vsimem/raster.tif', 'driver': 'tif', 'width': 4, 'height': 4, 'srid': 4326, 'bands': [{ 'data': range(16), }], }) # Do a round trip from raster to buffer to raster. result = GDALRaster(rast.vsi_buffer).bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to nodata value except on input block of ones. self.assertEqual(result, list(range(16))) # The vsi buffer is None for rasters that are not vsi based. self.assertIsNone(self.rs.vsi_buffer) def test_offset_size_and_shape_on_raster_creation(self): rast = GDALRaster({ 'datatype': 1, 'width': 4, 'height': 4, 'srid': 4326, 'bands': [{ 'data': (1,), 'offset': (1, 1), 'size': (2, 2), 'shape': (1, 1), 'nodata_value': 2, }], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to nodata value except on input block of ones. self.assertEqual( result, [2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2] ) def test_set_nodata_value_on_raster_creation(self): # Create raster filled with nodata values. rast = GDALRaster({ 'datatype': 1, 'width': 2, 'height': 2, 'srid': 4326, 'bands': [{'nodata_value': 23}], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # All band data is equal to nodata value. self.assertEqual(result, [23] * 4) def test_set_nodata_none_on_raster_creation(self): # Create raster without data and without nodata value. rast = GDALRaster({ 'datatype': 1, 'width': 2, 'height': 2, 'srid': 4326, 'bands': [{'nodata_value': None}], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to zero because no nodata value has been specified. self.assertEqual(result, [0] * 4) def test_raster_metadata_property(self): data = self.rs.metadata self.assertEqual(data['DEFAULT'], {'AREA_OR_POINT': 'Area'}) self.assertEqual(data['IMAGE_STRUCTURE'], {'INTERLEAVE': 'BAND'}) # Create file-based raster from scratch source = GDALRaster({ 'datatype': 1, 'width': 2, 'height': 2, 'srid': 4326, 'bands': [{'data': range(4), 'nodata_value': 99}], }) # Set metadata on raster and on a band. metadata = { 'DEFAULT': {'OWNER': 'Django', 'VERSION': '1.0', 'AREA_OR_POINT': 'Point'}, } source.metadata = metadata source.bands[0].metadata = metadata self.assertEqual(source.metadata['DEFAULT'], metadata['DEFAULT']) self.assertEqual(source.bands[0].metadata['DEFAULT'], metadata['DEFAULT']) # Update metadata on raster. metadata = { 'DEFAULT': {'VERSION': '2.0'}, } source.metadata = metadata self.assertEqual(source.metadata['DEFAULT']['VERSION'], '2.0') # Remove metadata on raster. metadata = { 'DEFAULT': {'OWNER': None}, } source.metadata = metadata self.assertNotIn('OWNER', source.metadata['DEFAULT']) def test_raster_info_accessor(self): infos = self.rs.info # Data info_lines = [line.strip() for line in infos.split('\n') if line.strip() != ''] for line in [ 'Driver: GTiff/GeoTIFF', 'Files: {}'.format(self.rs_path), 'Size is 163, 174', 'Origin = (511700.468070655711927,435103.377123198588379)', 'Pixel Size = (100.000000000000000,-100.000000000000000)', 'Metadata:', 'AREA_OR_POINT=Area', 'Image Structure Metadata:', 'INTERLEAVE=BAND', 'Band 1 Block=163x50 Type=Byte, ColorInterp=Gray', 'NoData Value=15' ]: self.assertIn(line, info_lines) for line in [ r'Upper Left \( 511700.468, 435103.377\) \( 82d51\'46.1\d"W, 27d55\' 1.5\d"N\)', r'Lower Left \( 511700.468, 417703.377\) \( 82d51\'52.0\d"W, 27d45\'37.5\d"N\)', r'Upper Right \( 528000.468, 435103.377\) \( 82d41\'48.8\d"W, 27d54\'56.3\d"N\)', r'Lower Right \( 528000.468, 417703.377\) \( 82d41\'55.5\d"W, 27d45\'32.2\d"N\)', r'Center \( 519850.468, 426403.377\) \( 82d46\'50.6\d"W, 27d50\'16.9\d"N\)', ]: self.assertRegex(infos, line) # CRS (skip the name because string depends on the GDAL/Proj versions). self.assertIn("NAD83 / Florida GDL Albers", infos) def test_compressed_file_based_raster_creation(self): rstfile = tempfile.NamedTemporaryFile(suffix='.tif') # Make a compressed copy of an existing raster. compressed = self.rs.warp({'papsz_options': {'compress': 'packbits'}, 'name': rstfile.name}) # Check physically if compression worked. self.assertLess(os.path.getsize(compressed.name), os.path.getsize(self.rs.name)) # Create file-based raster with options from scratch. compressed = GDALRaster({ 'datatype': 1, 'driver': 'tif', 'name': rstfile.name, 'width': 40, 'height': 40, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(40 ^ 2), 'nodata_value': 255, }], 'papsz_options': { 'compress': 'packbits', 'pixeltype': 'signedbyte', 'blockxsize': 23, 'blockysize': 23, } }) # Check if options used on creation are stored in metadata. # Reopening the raster ensures that all metadata has been written # to the file. compressed = GDALRaster(compressed.name) self.assertEqual(compressed.metadata['IMAGE_STRUCTURE']['COMPRESSION'], 'PACKBITS',) self.assertEqual(compressed.bands[0].metadata['IMAGE_STRUCTURE']['PIXELTYPE'], 'SIGNEDBYTE') self.assertIn('Block=40x23', compressed.info) def test_raster_warp(self): # Create in memory raster source = GDALRaster({ 'datatype': 1, 'driver': 'MEM', 'name': 'sourceraster', 'width': 4, 'height': 4, 'nr_of_bands': 1, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(16), 'nodata_value': 255, }], }) # Test altering the scale, width, and height of a raster data = { 'scale': [200, -200], 'width': 2, 'height': 2, } target = source.warp(data) self.assertEqual(target.width, data['width']) self.assertEqual(target.height, data['height']) self.assertEqual(target.scale, data['scale']) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertEqual(target.name, 'sourceraster_copy.MEM') result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [5, 7, 13, 15]) # Test altering the name and datatype (to float) data = { 'name': '/path/to/targetraster.tif', 'datatype': 6, } target = source.warp(data) self.assertEqual(target.bands[0].datatype(), 6) self.assertEqual(target.name, '/path/to/targetraster.tif') self.assertEqual(target.driver.name, 'MEM') result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual( result, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0] ) def test_raster_warp_nodata_zone(self): # Create in memory raster. source = GDALRaster({ 'datatype': 1, 'driver': 'MEM', 'width': 4, 'height': 4, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(16), 'nodata_value': 23, }], }) # Warp raster onto a location that does not cover any pixels of the original. result = source.warp({'origin': (200000, 200000)}).bands[0].data() if numpy: result = result.flatten().tolist() # The result is an empty raster filled with the correct nodata value. self.assertEqual(result, [23] * 16) def test_raster_clone(self): rstfile = tempfile.NamedTemporaryFile(suffix='.tif') tests = [ ('MEM', '', 23), # In memory raster. ('tif', rstfile.name, 99), # In file based raster. ] for driver, name, nodata_value in tests: with self.subTest(driver=driver): source = GDALRaster({ 'datatype': 1, 'driver': driver, 'name': name, 'width': 4, 'height': 4, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(16), 'nodata_value': nodata_value, }], }) clone = source.clone() self.assertNotEqual(clone.name, source.name) self.assertEqual(clone._write, source._write) self.assertEqual(clone.srs.srid, source.srs.srid) self.assertEqual(clone.width, source.width) self.assertEqual(clone.height, source.height) self.assertEqual(clone.origin, source.origin) self.assertEqual(clone.scale, source.scale) self.assertEqual(clone.skew, source.skew) self.assertIsNot(clone, source) def test_raster_transform(self): tests = [ 3086, '3086', SpatialReference(3086), ] for srs in tests: with self.subTest(srs=srs): # Prepare tempfile and nodata value. rstfile = tempfile.NamedTemporaryFile(suffix='.tif') ndv = 99 # Create in file based raster. source = GDALRaster({ 'datatype': 1, 'driver': 'tif', 'name': rstfile.name, 'width': 5, 'height': 5, 'nr_of_bands': 1, 'srid': 4326, 'origin': (-5, 5), 'scale': (2, -2), 'skew': (0, 0), 'bands': [{ 'data': range(25), 'nodata_value': ndv, }], }) target = source.transform(srs) # Reload data from disk. target = GDALRaster(target.name) self.assertEqual(target.srs.srid, 3086) self.assertEqual(target.width, 7) self.assertEqual(target.height, 7) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertAlmostEqual(target.origin[0], 9124842.791079799, 3) self.assertAlmostEqual(target.origin[1], 1589911.6476407414, 3) self.assertAlmostEqual(target.scale[0], 223824.82664250192, 3) self.assertAlmostEqual(target.scale[1], -223824.82664250192, 3) self.assertEqual(target.skew, [0, 0]) result = target.bands[0].data() if numpy: result = result.flatten().tolist() # The reprojection of a raster that spans over a large area # skews the data matrix and might introduce nodata values. self.assertEqual( result, [ ndv, ndv, ndv, ndv, 4, ndv, ndv, ndv, ndv, 2, 3, 9, ndv, ndv, ndv, 1, 2, 8, 13, 19, ndv, 0, 6, 6, 12, 18, 18, 24, ndv, 10, 11, 16, 22, 23, ndv, ndv, ndv, 15, 21, 22, ndv, ndv, ndv, ndv, 20, ndv, ndv, ndv, ndv, ], ) def test_raster_transform_clone(self): with mock.patch.object(GDALRaster, 'clone') as mocked_clone: # Create in file based raster. rstfile = tempfile.NamedTemporaryFile(suffix='.tif') source = GDALRaster({ 'datatype': 1, 'driver': 'tif', 'name': rstfile.name, 'width': 5, 'height': 5, 'nr_of_bands': 1, 'srid': 4326, 'origin': (-5, 5), 'scale': (2, -2), 'skew': (0, 0), 'bands': [{ 'data': range(25), 'nodata_value': 99, }], }) # transform() returns a clone because it is the same SRID and # driver. source.transform(4326) self.assertEqual(mocked_clone.call_count, 1) def test_raster_transform_clone_name(self): # Create in file based raster. rstfile = tempfile.NamedTemporaryFile(suffix='.tif') source = GDALRaster({ 'datatype': 1, 'driver': 'tif', 'name': rstfile.name, 'width': 5, 'height': 5, 'nr_of_bands': 1, 'srid': 4326, 'origin': (-5, 5), 'scale': (2, -2), 'skew': (0, 0), 'bands': [{ 'data': range(25), 'nodata_value': 99, }], }) clone_name = rstfile.name + '_respect_name.GTiff' target = source.transform(4326, name=clone_name) self.assertEqual(target.name, clone_name)
def test_raster_warp(self): # Create in memory raster source = GDALRaster({ "datatype": 1, "driver": "MEM", "name": "sourceraster", "width": 4, "height": 4, "nr_of_bands": 1, "srid": 3086, "origin": (500000, 400000), "scale": (100, -100), "skew": (0, 0), "bands": [{ "data": range(16), "nodata_value": 255, }], }) # Test altering the scale, width, and height of a raster data = { "scale": [200, -200], "width": 2, "height": 2, } target = source.warp(data) self.assertEqual(target.width, data["width"]) self.assertEqual(target.height, data["height"]) self.assertEqual(target.scale, data["scale"]) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertEqual(target.name, "sourceraster_copy.MEM") result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [5, 7, 13, 15]) # Test altering the name and datatype (to float) data = { "name": "/path/to/targetraster.tif", "datatype": 6, } target = source.warp(data) self.assertEqual(target.bands[0].datatype(), 6) self.assertEqual(target.name, "/path/to/targetraster.tif") self.assertEqual(target.driver.name, "MEM") result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual( result, [ 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, ], )
class GDALRasterTests(SimpleTestCase): """ Test a GDALRaster instance created from a file (GeoTiff). """ def setUp(self): self.rs_path = os.path.join(os.path.dirname(__file__), "../data/rasters/raster.tif") self.rs = GDALRaster(self.rs_path) def test_gdalraster_input_as_path(self): rs_path = Path( __file__).parent.parent / "data" / "rasters" / "raster.tif" rs = GDALRaster(rs_path) self.assertEqual(str(rs_path), rs.name) def test_rs_name_repr(self): self.assertEqual(self.rs_path, self.rs.name) self.assertRegex(repr(self.rs), r"<Raster object at 0x\w+>") def test_rs_driver(self): self.assertEqual(self.rs.driver.name, "GTiff") def test_rs_size(self): self.assertEqual(self.rs.width, 163) self.assertEqual(self.rs.height, 174) def test_rs_srs(self): self.assertEqual(self.rs.srs.srid, 3086) self.assertEqual(self.rs.srs.units, (1.0, "metre")) def test_rs_srid(self): rast = GDALRaster({ "width": 16, "height": 16, "srid": 4326, }) self.assertEqual(rast.srid, 4326) rast.srid = 3086 self.assertEqual(rast.srid, 3086) def test_geotransform_and_friends(self): # Assert correct values for file based raster self.assertEqual( self.rs.geotransform, [511700.4680706557, 100.0, 0.0, 435103.3771231986, 0.0, -100.0], ) self.assertEqual(self.rs.origin, [511700.4680706557, 435103.3771231986]) self.assertEqual(self.rs.origin.x, 511700.4680706557) self.assertEqual(self.rs.origin.y, 435103.3771231986) self.assertEqual(self.rs.scale, [100.0, -100.0]) self.assertEqual(self.rs.scale.x, 100.0) self.assertEqual(self.rs.scale.y, -100.0) self.assertEqual(self.rs.skew, [0, 0]) self.assertEqual(self.rs.skew.x, 0) self.assertEqual(self.rs.skew.y, 0) # Create in-memory rasters and change gtvalues rsmem = GDALRaster(JSON_RASTER) # geotransform accepts both floats and ints rsmem.geotransform = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] self.assertEqual(rsmem.geotransform, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]) rsmem.geotransform = range(6) self.assertEqual(rsmem.geotransform, [float(x) for x in range(6)]) self.assertEqual(rsmem.origin, [0, 3]) self.assertEqual(rsmem.origin.x, 0) self.assertEqual(rsmem.origin.y, 3) self.assertEqual(rsmem.scale, [1, 5]) self.assertEqual(rsmem.scale.x, 1) self.assertEqual(rsmem.scale.y, 5) self.assertEqual(rsmem.skew, [2, 4]) self.assertEqual(rsmem.skew.x, 2) self.assertEqual(rsmem.skew.y, 4) self.assertEqual(rsmem.width, 5) self.assertEqual(rsmem.height, 5) def test_geotransform_bad_inputs(self): rsmem = GDALRaster(JSON_RASTER) error_geotransforms = [ [1, 2], [1, 2, 3, 4, 5, "foo"], [1, 2, 3, 4, 5, 6, "foo"], ] msg = "Geotransform must consist of 6 numeric values." for geotransform in error_geotransforms: with self.subTest(i=geotransform), self.assertRaisesMessage( ValueError, msg): rsmem.geotransform = geotransform def test_rs_extent(self): self.assertEqual( self.rs.extent, ( 511700.4680706557, 417703.3771231986, 528000.4680706557, 435103.3771231986, ), ) def test_rs_bands(self): self.assertEqual(len(self.rs.bands), 1) self.assertIsInstance(self.rs.bands[0], GDALBand) def test_memory_based_raster_creation(self): # Create uint8 raster with full pixel data range (0-255) rast = GDALRaster({ "datatype": 1, "width": 16, "height": 16, "srid": 4326, "bands": [{ "data": range(256), "nodata_value": 255, }], }) # Get array from raster result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Assert data is same as original input self.assertEqual(result, list(range(256))) def test_file_based_raster_creation(self): # Prepare tempfile rstfile = tempfile.NamedTemporaryFile(suffix=".tif") # Create file-based raster from scratch GDALRaster({ "datatype": self.rs.bands[0].datatype(), "driver": "tif", "name": rstfile.name, "width": 163, "height": 174, "nr_of_bands": 1, "srid": self.rs.srs.wkt, "origin": (self.rs.origin.x, self.rs.origin.y), "scale": (self.rs.scale.x, self.rs.scale.y), "skew": (self.rs.skew.x, self.rs.skew.y), "bands": [{ "data": self.rs.bands[0].data(), "nodata_value": self.rs.bands[0].nodata_value, }], }) # Reload newly created raster from file restored_raster = GDALRaster(rstfile.name) # Presence of TOWGS84 depend on GDAL/Proj versions. self.assertEqual( restored_raster.srs.wkt.replace("TOWGS84[0,0,0,0,0,0,0],", ""), self.rs.srs.wkt.replace("TOWGS84[0,0,0,0,0,0,0],", ""), ) self.assertEqual(restored_raster.geotransform, self.rs.geotransform) if numpy: numpy.testing.assert_equal(restored_raster.bands[0].data(), self.rs.bands[0].data()) else: self.assertEqual(restored_raster.bands[0].data(), self.rs.bands[0].data()) def test_nonexistent_file(self): msg = 'Unable to read raster source input "nonexistent.tif".' with self.assertRaisesMessage(GDALException, msg): GDALRaster("nonexistent.tif") def test_vsi_raster_creation(self): # Open a raster as a file object. with open(self.rs_path, "rb") as dat: # Instantiate a raster from the file binary buffer. vsimem = GDALRaster(dat.read()) # The data of the in-memory file is equal to the source file. result = vsimem.bands[0].data() target = self.rs.bands[0].data() if numpy: result = result.flatten().tolist() target = target.flatten().tolist() self.assertEqual(result, target) def test_vsi_raster_deletion(self): path = "/vsimem/raster.tif" # Create a vsi-based raster from scratch. vsimem = GDALRaster({ "name": path, "driver": "tif", "width": 4, "height": 4, "srid": 4326, "bands": [{ "data": range(16), }], }) # The virtual file exists. rst = GDALRaster(path) self.assertEqual(rst.width, 4) # Delete GDALRaster. del vsimem del rst # The virtual file has been removed. msg = 'Could not open the datasource at "/vsimem/raster.tif"' with self.assertRaisesMessage(GDALException, msg): GDALRaster(path) def test_vsi_invalid_buffer_error(self): msg = "Failed creating VSI raster from the input buffer." with self.assertRaisesMessage(GDALException, msg): GDALRaster(b"not-a-raster-buffer") def test_vsi_buffer_property(self): # Create a vsi-based raster from scratch. rast = GDALRaster({ "name": "/vsimem/raster.tif", "driver": "tif", "width": 4, "height": 4, "srid": 4326, "bands": [{ "data": range(16), }], }) # Do a round trip from raster to buffer to raster. result = GDALRaster(rast.vsi_buffer).bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to nodata value except on input block of ones. self.assertEqual(result, list(range(16))) # The vsi buffer is None for rasters that are not vsi based. self.assertIsNone(self.rs.vsi_buffer) def test_vsi_vsizip_filesystem(self): rst_zipfile = tempfile.NamedTemporaryFile(suffix=".zip") with zipfile.ZipFile(rst_zipfile, mode="w") as zf: zf.write(self.rs_path, "raster.tif") rst_path = "/vsizip/" + os.path.join(rst_zipfile.name, "raster.tif") rst = GDALRaster(rst_path) self.assertEqual(rst.driver.name, self.rs.driver.name) self.assertEqual(rst.name, rst_path) self.assertIs(rst.is_vsi_based, True) self.assertIsNone(rst.vsi_buffer) def test_offset_size_and_shape_on_raster_creation(self): rast = GDALRaster({ "datatype": 1, "width": 4, "height": 4, "srid": 4326, "bands": [{ "data": (1, ), "offset": (1, 1), "size": (2, 2), "shape": (1, 1), "nodata_value": 2, }], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to nodata value except on input block of ones. self.assertEqual(result, [2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2]) def test_set_nodata_value_on_raster_creation(self): # Create raster filled with nodata values. rast = GDALRaster({ "datatype": 1, "width": 2, "height": 2, "srid": 4326, "bands": [{ "nodata_value": 23 }], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # All band data is equal to nodata value. self.assertEqual(result, [23] * 4) def test_set_nodata_none_on_raster_creation(self): # Create raster without data and without nodata value. rast = GDALRaster({ "datatype": 1, "width": 2, "height": 2, "srid": 4326, "bands": [{ "nodata_value": None }], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to zero because no nodata value has been specified. self.assertEqual(result, [0] * 4) def test_raster_metadata_property(self): data = self.rs.metadata self.assertEqual(data["DEFAULT"], {"AREA_OR_POINT": "Area"}) self.assertEqual(data["IMAGE_STRUCTURE"], {"INTERLEAVE": "BAND"}) # Create file-based raster from scratch source = GDALRaster({ "datatype": 1, "width": 2, "height": 2, "srid": 4326, "bands": [{ "data": range(4), "nodata_value": 99 }], }) # Set metadata on raster and on a band. metadata = { "DEFAULT": { "OWNER": "Django", "VERSION": "1.0", "AREA_OR_POINT": "Point" }, } source.metadata = metadata source.bands[0].metadata = metadata self.assertEqual(source.metadata["DEFAULT"], metadata["DEFAULT"]) self.assertEqual(source.bands[0].metadata["DEFAULT"], metadata["DEFAULT"]) # Update metadata on raster. metadata = { "DEFAULT": { "VERSION": "2.0" }, } source.metadata = metadata self.assertEqual(source.metadata["DEFAULT"]["VERSION"], "2.0") # Remove metadata on raster. metadata = { "DEFAULT": { "OWNER": None }, } source.metadata = metadata self.assertNotIn("OWNER", source.metadata["DEFAULT"]) def test_raster_info_accessor(self): infos = self.rs.info # Data info_lines = [ line.strip() for line in infos.split("\n") if line.strip() != "" ] for line in [ "Driver: GTiff/GeoTIFF", "Files: {}".format(self.rs_path), "Size is 163, 174", "Origin = (511700.468070655711927,435103.377123198588379)", "Pixel Size = (100.000000000000000,-100.000000000000000)", "Metadata:", "AREA_OR_POINT=Area", "Image Structure Metadata:", "INTERLEAVE=BAND", "Band 1 Block=163x50 Type=Byte, ColorInterp=Gray", "NoData Value=15", ]: self.assertIn(line, info_lines) for line in [ r"Upper Left \( 511700.468, 435103.377\) " r'\( 82d51\'46.1\d"W, 27d55\' 1.5\d"N\)', r"Lower Left \( 511700.468, 417703.377\) " r'\( 82d51\'52.0\d"W, 27d45\'37.5\d"N\)', r"Upper Right \( 528000.468, 435103.377\) " r'\( 82d41\'48.8\d"W, 27d54\'56.3\d"N\)', r"Lower Right \( 528000.468, 417703.377\) " r'\( 82d41\'55.5\d"W, 27d45\'32.2\d"N\)', r"Center \( 519850.468, 426403.377\) " r'\( 82d46\'50.6\d"W, 27d50\'16.9\d"N\)', ]: self.assertRegex(infos, line) # CRS (skip the name because string depends on the GDAL/Proj versions). self.assertIn("NAD83 / Florida GDL Albers", infos) def test_compressed_file_based_raster_creation(self): rstfile = tempfile.NamedTemporaryFile(suffix=".tif") # Make a compressed copy of an existing raster. compressed = self.rs.warp({ "papsz_options": { "compress": "packbits" }, "name": rstfile.name }) # Check physically if compression worked. self.assertLess(os.path.getsize(compressed.name), os.path.getsize(self.rs.name)) # Create file-based raster with options from scratch. compressed = GDALRaster({ "datatype": 1, "driver": "tif", "name": rstfile.name, "width": 40, "height": 40, "srid": 3086, "origin": (500000, 400000), "scale": (100, -100), "skew": (0, 0), "bands": [{ "data": range(40 ^ 2), "nodata_value": 255, }], "papsz_options": { "compress": "packbits", "pixeltype": "signedbyte", "blockxsize": 23, "blockysize": 23, }, }) # Check if options used on creation are stored in metadata. # Reopening the raster ensures that all metadata has been written # to the file. compressed = GDALRaster(compressed.name) self.assertEqual( compressed.metadata["IMAGE_STRUCTURE"]["COMPRESSION"], "PACKBITS", ) self.assertEqual( compressed.bands[0].metadata["IMAGE_STRUCTURE"]["PIXELTYPE"], "SIGNEDBYTE") self.assertIn("Block=40x23", compressed.info) def test_raster_warp(self): # Create in memory raster source = GDALRaster({ "datatype": 1, "driver": "MEM", "name": "sourceraster", "width": 4, "height": 4, "nr_of_bands": 1, "srid": 3086, "origin": (500000, 400000), "scale": (100, -100), "skew": (0, 0), "bands": [{ "data": range(16), "nodata_value": 255, }], }) # Test altering the scale, width, and height of a raster data = { "scale": [200, -200], "width": 2, "height": 2, } target = source.warp(data) self.assertEqual(target.width, data["width"]) self.assertEqual(target.height, data["height"]) self.assertEqual(target.scale, data["scale"]) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertEqual(target.name, "sourceraster_copy.MEM") result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [5, 7, 13, 15]) # Test altering the name and datatype (to float) data = { "name": "/path/to/targetraster.tif", "datatype": 6, } target = source.warp(data) self.assertEqual(target.bands[0].datatype(), 6) self.assertEqual(target.name, "/path/to/targetraster.tif") self.assertEqual(target.driver.name, "MEM") result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual( result, [ 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, ], ) def test_raster_warp_nodata_zone(self): # Create in memory raster. source = GDALRaster({ "datatype": 1, "driver": "MEM", "width": 4, "height": 4, "srid": 3086, "origin": (500000, 400000), "scale": (100, -100), "skew": (0, 0), "bands": [{ "data": range(16), "nodata_value": 23, }], }) # Warp raster onto a location that does not cover any pixels of the original. result = source.warp({"origin": (200000, 200000)}).bands[0].data() if numpy: result = result.flatten().tolist() # The result is an empty raster filled with the correct nodata value. self.assertEqual(result, [23] * 16) def test_raster_clone(self): rstfile = tempfile.NamedTemporaryFile(suffix=".tif") tests = [ ("MEM", "", 23), # In memory raster. ("tif", rstfile.name, 99), # In file based raster. ] for driver, name, nodata_value in tests: with self.subTest(driver=driver): source = GDALRaster({ "datatype": 1, "driver": driver, "name": name, "width": 4, "height": 4, "srid": 3086, "origin": (500000, 400000), "scale": (100, -100), "skew": (0, 0), "bands": [{ "data": range(16), "nodata_value": nodata_value, }], }) clone = source.clone() self.assertNotEqual(clone.name, source.name) self.assertEqual(clone._write, source._write) self.assertEqual(clone.srs.srid, source.srs.srid) self.assertEqual(clone.width, source.width) self.assertEqual(clone.height, source.height) self.assertEqual(clone.origin, source.origin) self.assertEqual(clone.scale, source.scale) self.assertEqual(clone.skew, source.skew) self.assertIsNot(clone, source) def test_raster_transform(self): tests = [ 3086, "3086", SpatialReference(3086), ] for srs in tests: with self.subTest(srs=srs): # Prepare tempfile and nodata value. rstfile = tempfile.NamedTemporaryFile(suffix=".tif") ndv = 99 # Create in file based raster. source = GDALRaster({ "datatype": 1, "driver": "tif", "name": rstfile.name, "width": 5, "height": 5, "nr_of_bands": 1, "srid": 4326, "origin": (-5, 5), "scale": (2, -2), "skew": (0, 0), "bands": [{ "data": range(25), "nodata_value": ndv, }], }) target = source.transform(srs) # Reload data from disk. target = GDALRaster(target.name) self.assertEqual(target.srs.srid, 3086) self.assertEqual(target.width, 7) self.assertEqual(target.height, 7) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertAlmostEqual(target.origin[0], 9124842.791079799, 3) self.assertAlmostEqual(target.origin[1], 1589911.6476407414, 3) self.assertAlmostEqual(target.scale[0], 223824.82664250192, 3) self.assertAlmostEqual(target.scale[1], -223824.82664250192, 3) self.assertEqual(target.skew, [0, 0]) result = target.bands[0].data() if numpy: result = result.flatten().tolist() # The reprojection of a raster that spans over a large area # skews the data matrix and might introduce nodata values. self.assertEqual( result, [ ndv, ndv, ndv, ndv, 4, ndv, ndv, ndv, ndv, 2, 3, 9, ndv, ndv, ndv, 1, 2, 8, 13, 19, ndv, 0, 6, 6, 12, 18, 18, 24, ndv, 10, 11, 16, 22, 23, ndv, ndv, ndv, 15, 21, 22, ndv, ndv, ndv, ndv, 20, ndv, ndv, ndv, ndv, ], ) def test_raster_transform_clone(self): with mock.patch.object(GDALRaster, "clone") as mocked_clone: # Create in file based raster. rstfile = tempfile.NamedTemporaryFile(suffix=".tif") source = GDALRaster({ "datatype": 1, "driver": "tif", "name": rstfile.name, "width": 5, "height": 5, "nr_of_bands": 1, "srid": 4326, "origin": (-5, 5), "scale": (2, -2), "skew": (0, 0), "bands": [{ "data": range(25), "nodata_value": 99, }], }) # transform() returns a clone because it is the same SRID and # driver. source.transform(4326) self.assertEqual(mocked_clone.call_count, 1) def test_raster_transform_clone_name(self): # Create in file based raster. rstfile = tempfile.NamedTemporaryFile(suffix=".tif") source = GDALRaster({ "datatype": 1, "driver": "tif", "name": rstfile.name, "width": 5, "height": 5, "nr_of_bands": 1, "srid": 4326, "origin": (-5, 5), "scale": (2, -2), "skew": (0, 0), "bands": [{ "data": range(25), "nodata_value": 99, }], }) clone_name = rstfile.name + "_respect_name.GTiff" target = source.transform(4326, name=clone_name) self.assertEqual(target.name, clone_name)
class GDALRasterTests(SimpleTestCase): """ Test a GDALRaster instance created from a file (GeoTiff). """ def setUp(self): self.rs_path = os.path.join(os.path.dirname(__file__), '../data/rasters/raster.tif') self.rs = GDALRaster(self.rs_path) def test_rs_name_repr(self): self.assertEqual(self.rs_path, self.rs.name) self.assertRegex(repr(self.rs), r"<Raster object at 0x\w+>") def test_rs_driver(self): self.assertEqual(self.rs.driver.name, 'GTiff') def test_rs_size(self): self.assertEqual(self.rs.width, 163) self.assertEqual(self.rs.height, 174) def test_rs_srs(self): self.assertEqual(self.rs.srs.srid, 3086) self.assertEqual(self.rs.srs.units, (1.0, 'metre')) def test_rs_srid(self): rast = GDALRaster({ 'width': 16, 'height': 16, 'srid': 4326, }) self.assertEqual(rast.srid, 4326) rast.srid = 3086 self.assertEqual(rast.srid, 3086) def test_geotransform_and_friends(self): # Assert correct values for file based raster self.assertEqual( self.rs.geotransform, [511700.4680706557, 100.0, 0.0, 435103.3771231986, 0.0, -100.0] ) self.assertEqual(self.rs.origin, [511700.4680706557, 435103.3771231986]) self.assertEqual(self.rs.origin.x, 511700.4680706557) self.assertEqual(self.rs.origin.y, 435103.3771231986) self.assertEqual(self.rs.scale, [100.0, -100.0]) self.assertEqual(self.rs.scale.x, 100.0) self.assertEqual(self.rs.scale.y, -100.0) self.assertEqual(self.rs.skew, [0, 0]) self.assertEqual(self.rs.skew.x, 0) self.assertEqual(self.rs.skew.y, 0) # Create in-memory rasters and change gtvalues rsmem = GDALRaster(JSON_RASTER) # geotransform accepts both floats and ints rsmem.geotransform = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0] self.assertEqual(rsmem.geotransform, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]) rsmem.geotransform = range(6) self.assertEqual(rsmem.geotransform, [float(x) for x in range(6)]) self.assertEqual(rsmem.origin, [0, 3]) self.assertEqual(rsmem.origin.x, 0) self.assertEqual(rsmem.origin.y, 3) self.assertEqual(rsmem.scale, [1, 5]) self.assertEqual(rsmem.scale.x, 1) self.assertEqual(rsmem.scale.y, 5) self.assertEqual(rsmem.skew, [2, 4]) self.assertEqual(rsmem.skew.x, 2) self.assertEqual(rsmem.skew.y, 4) self.assertEqual(rsmem.width, 5) self.assertEqual(rsmem.height, 5) def test_geotransform_bad_inputs(self): rsmem = GDALRaster(JSON_RASTER) error_geotransforms = [ [1, 2], [1, 2, 3, 4, 5, 'foo'], [1, 2, 3, 4, 5, 6, 'foo'], ] msg = 'Geotransform must consist of 6 numeric values.' for geotransform in error_geotransforms: with self.subTest(i=geotransform), self.assertRaisesMessage(ValueError, msg): rsmem.geotransform = geotransform def test_rs_extent(self): self.assertEqual( self.rs.extent, (511700.4680706557, 417703.3771231986, 528000.4680706557, 435103.3771231986) ) def test_rs_bands(self): self.assertEqual(len(self.rs.bands), 1) self.assertIsInstance(self.rs.bands[0], GDALBand) def test_memory_based_raster_creation(self): # Create uint8 raster with full pixel data range (0-255) rast = GDALRaster({ 'datatype': 1, 'width': 16, 'height': 16, 'srid': 4326, 'bands': [{ 'data': range(256), 'nodata_value': 255, }], }) # Get array from raster result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Assert data is same as original input self.assertEqual(result, list(range(256))) def test_file_based_raster_creation(self): # Prepare tempfile rstfile = tempfile.NamedTemporaryFile(suffix='.tif') # Create file-based raster from scratch GDALRaster({ 'datatype': self.rs.bands[0].datatype(), 'driver': 'tif', 'name': rstfile.name, 'width': 163, 'height': 174, 'nr_of_bands': 1, 'srid': self.rs.srs.wkt, 'origin': (self.rs.origin.x, self.rs.origin.y), 'scale': (self.rs.scale.x, self.rs.scale.y), 'skew': (self.rs.skew.x, self.rs.skew.y), 'bands': [{ 'data': self.rs.bands[0].data(), 'nodata_value': self.rs.bands[0].nodata_value, }], }) # Reload newly created raster from file restored_raster = GDALRaster(rstfile.name) self.assertEqual(restored_raster.srs.wkt, self.rs.srs.wkt) self.assertEqual(restored_raster.geotransform, self.rs.geotransform) if numpy: numpy.testing.assert_equal( restored_raster.bands[0].data(), self.rs.bands[0].data() ) else: self.assertEqual(restored_raster.bands[0].data(), self.rs.bands[0].data()) def test_vsi_raster_creation(self): # Open a raster as a file object. with open(self.rs_path, 'rb') as dat: # Instantiate a raster from the file binary buffer. vsimem = GDALRaster(dat.read()) # The data of the in-memory file is equal to the source file. result = vsimem.bands[0].data() target = self.rs.bands[0].data() if numpy: result = result.flatten().tolist() target = target.flatten().tolist() self.assertEqual(result, target) def test_vsi_raster_deletion(self): path = '/vsimem/raster.tif' # Create a vsi-based raster from scratch. vsimem = GDALRaster({ 'name': path, 'driver': 'tif', 'width': 4, 'height': 4, 'srid': 4326, 'bands': [{ 'data': range(16), }], }) # The virtual file exists. rst = GDALRaster(path) self.assertEqual(rst.width, 4) # Delete GDALRaster. del vsimem del rst # The virtual file has been removed. msg = 'Could not open the datasource at "/vsimem/raster.tif"' with self.assertRaisesMessage(GDALException, msg): GDALRaster(path) def test_vsi_invalid_buffer_error(self): msg = 'Failed creating VSI raster from the input buffer.' with self.assertRaisesMessage(GDALException, msg): GDALRaster(b'not-a-raster-buffer') def test_vsi_buffer_property(self): # Create a vsi-based raster from scratch. rast = GDALRaster({ 'name': '/vsimem/raster.tif', 'driver': 'tif', 'width': 4, 'height': 4, 'srid': 4326, 'bands': [{ 'data': range(16), }], }) # Do a round trip from raster to buffer to raster. result = GDALRaster(rast.vsi_buffer).bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to nodata value except on input block of ones. self.assertEqual(result, list(range(16))) # The vsi buffer is None for rasters that are not vsi based. self.assertIsNone(self.rs.vsi_buffer) def test_offset_size_and_shape_on_raster_creation(self): rast = GDALRaster({ 'datatype': 1, 'width': 4, 'height': 4, 'srid': 4326, 'bands': [{ 'data': (1,), 'offset': (1, 1), 'size': (2, 2), 'shape': (1, 1), 'nodata_value': 2, }], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to nodata value except on input block of ones. self.assertEqual( result, [2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2] ) def test_set_nodata_value_on_raster_creation(self): # Create raster filled with nodata values. rast = GDALRaster({ 'datatype': 1, 'width': 2, 'height': 2, 'srid': 4326, 'bands': [{'nodata_value': 23}], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # All band data is equal to nodata value. self.assertEqual(result, [23] * 4) def test_set_nodata_none_on_raster_creation(self): if GDAL_VERSION < (2, 1): self.skipTest("GDAL >= 2.1 is required for this test.") # Create raster without data and without nodata value. rast = GDALRaster({ 'datatype': 1, 'width': 2, 'height': 2, 'srid': 4326, 'bands': [{'nodata_value': None}], }) # Get array from raster. result = rast.bands[0].data() if numpy: result = result.flatten().tolist() # Band data is equal to zero because no nodata value has been specified. self.assertEqual(result, [0] * 4) def test_raster_metadata_property(self): data = self.rs.metadata self.assertEqual(data['DEFAULT'], {'AREA_OR_POINT': 'Area'}) self.assertEqual(data['IMAGE_STRUCTURE'], {'INTERLEAVE': 'BAND'}) # Create file-based raster from scratch source = GDALRaster({ 'datatype': 1, 'width': 2, 'height': 2, 'srid': 4326, 'bands': [{'data': range(4), 'nodata_value': 99}], }) # Set metadata on raster and on a band. metadata = { 'DEFAULT': {'OWNER': 'Django', 'VERSION': '1.0', 'AREA_OR_POINT': 'Point'}, } source.metadata = metadata source.bands[0].metadata = metadata self.assertEqual(source.metadata['DEFAULT'], metadata['DEFAULT']) self.assertEqual(source.bands[0].metadata['DEFAULT'], metadata['DEFAULT']) # Update metadata on raster. metadata = { 'DEFAULT': {'VERSION': '2.0'}, } source.metadata = metadata self.assertEqual(source.metadata['DEFAULT']['VERSION'], '2.0') # Remove metadata on raster. metadata = { 'DEFAULT': {'OWNER': None}, } source.metadata = metadata self.assertNotIn('OWNER', source.metadata['DEFAULT']) def test_raster_info_accessor(self): if GDAL_VERSION < (2, 1): msg = 'GDAL ≥ 2.1 is required for using the info property.' with self.assertRaisesMessage(ValueError, msg): self.rs.info return gdalinfo = """ Driver: GTiff/GeoTIFF Files: {0} Size is 163, 174 Coordinate System is: PROJCS["NAD83 / Florida GDL Albers", GEOGCS["NAD83", DATUM["North_American_Datum_1983", SPHEROID["GRS 1980",6378137,298.257222101, AUTHORITY["EPSG","7019"]], TOWGS84[0,0,0,0,0,0,0], AUTHORITY["EPSG","6269"]], PRIMEM["Greenwich",0, AUTHORITY["EPSG","8901"]], UNIT["degree",0.0174532925199433, AUTHORITY["EPSG","9122"]], AUTHORITY["EPSG","4269"]], PROJECTION["Albers_Conic_Equal_Area"], PARAMETER["standard_parallel_1",24], PARAMETER["standard_parallel_2",31.5], PARAMETER["latitude_of_center",24], PARAMETER["longitude_of_center",-84], PARAMETER["false_easting",400000], PARAMETER["false_northing",0], UNIT["metre",1, AUTHORITY["EPSG","9001"]], AXIS["X",EAST], AXIS["Y",NORTH], AUTHORITY["EPSG","3086"]] Origin = (511700.468070655711927,435103.377123198588379) Pixel Size = (100.000000000000000,-100.000000000000000) Metadata: AREA_OR_POINT=Area Image Structure Metadata: INTERLEAVE=BAND Corner Coordinates: Upper Left ( 511700.468, 435103.377) ( 82d51'46.16"W, 27d55' 1.53"N) Lower Left ( 511700.468, 417703.377) ( 82d51'52.04"W, 27d45'37.50"N) Upper Right ( 528000.468, 435103.377) ( 82d41'48.81"W, 27d54'56.30"N) Lower Right ( 528000.468, 417703.377) ( 82d41'55.54"W, 27d45'32.28"N) Center ( 519850.468, 426403.377) ( 82d46'50.64"W, 27d50'16.99"N) Band 1 Block=163x50 Type=Byte, ColorInterp=Gray NoData Value=15 """.format(self.rs_path) # Data info_dyn = [line.strip() for line in self.rs.info.split('\n') if line.strip() != ''] info_ref = [line.strip() for line in gdalinfo.split('\n') if line.strip() != ''] self.assertEqual(info_dyn, info_ref) def test_compressed_file_based_raster_creation(self): rstfile = tempfile.NamedTemporaryFile(suffix='.tif') # Make a compressed copy of an existing raster. compressed = self.rs.warp({'papsz_options': {'compress': 'packbits'}, 'name': rstfile.name}) # Check physically if compression worked. self.assertLess(os.path.getsize(compressed.name), os.path.getsize(self.rs.name)) # Create file-based raster with options from scratch. compressed = GDALRaster({ 'datatype': 1, 'driver': 'tif', 'name': rstfile.name, 'width': 40, 'height': 40, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(40 ^ 2), 'nodata_value': 255, }], 'papsz_options': { 'compress': 'packbits', 'pixeltype': 'signedbyte', 'blockxsize': 23, 'blockysize': 23, } }) # Check if options used on creation are stored in metadata. # Reopening the raster ensures that all metadata has been written # to the file. compressed = GDALRaster(compressed.name) self.assertEqual(compressed.metadata['IMAGE_STRUCTURE']['COMPRESSION'], 'PACKBITS',) self.assertEqual(compressed.bands[0].metadata['IMAGE_STRUCTURE']['PIXELTYPE'], 'SIGNEDBYTE') if GDAL_VERSION >= (2, 1): self.assertIn('Block=40x23', compressed.info) def test_raster_warp(self): # Create in memory raster source = GDALRaster({ 'datatype': 1, 'driver': 'MEM', 'name': 'sourceraster', 'width': 4, 'height': 4, 'nr_of_bands': 1, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(16), 'nodata_value': 255, }], }) # Test altering the scale, width, and height of a raster data = { 'scale': [200, -200], 'width': 2, 'height': 2, } target = source.warp(data) self.assertEqual(target.width, data['width']) self.assertEqual(target.height, data['height']) self.assertEqual(target.scale, data['scale']) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertEqual(target.name, 'sourceraster_copy.MEM') result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual(result, [5, 7, 13, 15]) # Test altering the name and datatype (to float) data = { 'name': '/path/to/targetraster.tif', 'datatype': 6, } target = source.warp(data) self.assertEqual(target.bands[0].datatype(), 6) self.assertEqual(target.name, '/path/to/targetraster.tif') self.assertEqual(target.driver.name, 'MEM') result = target.bands[0].data() if numpy: result = result.flatten().tolist() self.assertEqual( result, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0] ) def test_raster_warp_nodata_zone(self): # Create in memory raster. source = GDALRaster({ 'datatype': 1, 'driver': 'MEM', 'width': 4, 'height': 4, 'srid': 3086, 'origin': (500000, 400000), 'scale': (100, -100), 'skew': (0, 0), 'bands': [{ 'data': range(16), 'nodata_value': 23, }], }) # Warp raster onto a location that does not cover any pixels of the original. result = source.warp({'origin': (200000, 200000)}).bands[0].data() if numpy: result = result.flatten().tolist() # The result is an empty raster filled with the correct nodata value. self.assertEqual(result, [23] * 16) def test_raster_transform(self): # Prepare tempfile and nodata value rstfile = tempfile.NamedTemporaryFile(suffix='.tif') ndv = 99 # Create in file based raster source = GDALRaster({ 'datatype': 1, 'driver': 'tif', 'name': rstfile.name, 'width': 5, 'height': 5, 'nr_of_bands': 1, 'srid': 4326, 'origin': (-5, 5), 'scale': (2, -2), 'skew': (0, 0), 'bands': [{ 'data': range(25), 'nodata_value': ndv, }], }) # Transform raster into srid 4326. target = source.transform(3086) # Reload data from disk target = GDALRaster(target.name) self.assertEqual(target.srs.srid, 3086) self.assertEqual(target.width, 7) self.assertEqual(target.height, 7) self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype()) self.assertAlmostEqual(target.origin[0], 9124842.791079799, 3) self.assertAlmostEqual(target.origin[1], 1589911.6476407414, 3) self.assertAlmostEqual(target.scale[0], 223824.82664250192, 3) self.assertAlmostEqual(target.scale[1], -223824.82664250192, 3) self.assertEqual(target.skew, [0, 0]) result = target.bands[0].data() if numpy: result = result.flatten().tolist() # The reprojection of a raster that spans over a large area # skews the data matrix and might introduce nodata values. self.assertEqual( result, [ ndv, ndv, ndv, ndv, 4, ndv, ndv, ndv, ndv, 2, 3, 9, ndv, ndv, ndv, 1, 2, 8, 13, 19, ndv, 0, 6, 6, 12, 18, 18, 24, ndv, 10, 11, 16, 22, 23, ndv, ndv, ndv, 15, 21, 22, ndv, ndv, ndv, ndv, 20, ndv, ndv, ndv, ndv, ] )
class RasterLayerParser(object): """ Class to parse raster layers. """ def __init__(self, rasterlayer): self.rasterlayer = rasterlayer self.rastername = os.path.basename(rasterlayer.rasterfile.name) # Set raster tilesize self.tilesize = int(getattr(settings, "RASTER_TILESIZE", WEB_MERCATOR_TILESIZE)) self.zoomdown = getattr(settings, "RASTER_ZOOM_NEXT_HIGHER", True) def log(self, msg, reset=False, status=None, zoom=None): """ Write a message to the parse log of the rasterlayer instance and update the parse status object. """ if status is not None: self.rasterlayer.parsestatus.status = status if zoom is not None: self.rasterlayer.parsestatus.tile_level = zoom # Prepare datetime stamp for log now = "[{0}] ".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) # Write log, reset if requested if reset: self.rasterlayer.parsestatus.log = now + msg else: self.rasterlayer.parsestatus.log += "\n" + now + msg self.rasterlayer.save() self.rasterlayer.parsestatus.save() def get_raster_file(self): """ Make local copy of rasterfile, which is needed if files are stored on remote storage, and unzip it if necessary. """ self.log("Getting raster file from storage") raster_workdir = getattr(settings, "RASTER_WORKDIR", None) self.tmpdir = tempfile.mkdtemp(dir=raster_workdir) # Access rasterfile and store in a temp folder rasterfile = open(os.path.join(self.tmpdir, self.rastername), "wb") for chunk in self.rasterlayer.rasterfile.chunks(): rasterfile.write(chunk) rasterfile.close() # If the raster file is compressed, decompress it fileName, fileExtension = os.path.splitext(self.rastername) if fileExtension == ".zip": # Open and extract zipfile zf = zipfile.ZipFile(os.path.join(self.tmpdir, self.rastername)) zf.extractall(self.tmpdir) # Remove zipfile os.remove(os.path.join(self.tmpdir, self.rastername)) # Get filelist from directory raster_list = glob.glob(os.path.join(self.tmpdir, "*.*")) # Check if only one file is found in zipfile if len(raster_list) > 1: self.log( "WARNING: Found more than one file in zipfile " "using only first file found. This might lead " "to problems if its not a raster file." ) # Return first one as raster file self.rastername = os.path.basename(raster_list[0]) def open_raster_file(self): """ Open the raster file as GDALRaster and set nodata-values. """ self.log("Opening raster file as GDALRaster.") # Open raster file self.dataset = GDALRaster(os.path.join(self.tmpdir, self.rastername), write=True) # Make sure nodata value is set from input self.hist_values = [] self.hist_bins = [] for i, band in enumerate(self.dataset.bands): if self.rasterlayer.nodata is not None: band.nodata_value = float(self.rasterlayer.nodata) # Create band metatdata object bandmeta = RasterLayerBandMetadata.objects.create( rasterlayer=self.rasterlayer, band=i, nodata_value=band.nodata_value, min=band.min, max=band.max ) # Prepare numpy hist values and bins self.hist_values.append(numpy.array(bandmeta.hist_values)) self.hist_bins.append(numpy.array(bandmeta.hist_bins)) # Store original metadata for this raster meta = self.rasterlayer.metadata meta.uperleftx = self.dataset.origin.x meta.uperlefty = self.dataset.origin.y meta.width = self.dataset.width meta.height = self.dataset.height meta.scalex = self.dataset.scale.x meta.scaley = self.dataset.scale.y meta.skewx = self.dataset.skew.x meta.skewy = self.dataset.skew.y meta.numbands = len(self.dataset.bands) meta.srs_wkt = self.dataset.srs.wkt meta.srid = self.dataset.srs.srid meta.save() def close_raster_file(self): """ On Windows close and release the GDALRaster resources """ try: if self.dataset: del self.dataset self.dataset = None except AttributeError: pass def create_tiles(self, zoom): """ Create tiles for this raster at the given zoomlevel. This routine first snaps the raster to the grid of the zoomlevel, then creates the tiles from the snapped raster. """ # Compute the tile x-y-z index range for the rasterlayer for this zoomlevel bbox = self.rasterlayer.extent() indexrange = tiler.tile_index_range(bbox, zoom) # Compute scale of tiles for this zoomlevel tilescale = tiler.tile_scale(zoom) # Count the number of tiles that are required to cover the raster at this zoomlevel nr_of_tiles = (indexrange[2] - indexrange[0] + 1) * (indexrange[3] - indexrange[1] + 1) # Create destination raster file self.log("Snapping dataset to zoom level {0}".format(zoom)) bounds = tiler.tile_bounds(indexrange[0], indexrange[1], zoom) sizex = (indexrange[2] - indexrange[0] + 1) * self.tilesize sizey = (indexrange[3] - indexrange[1] + 1) * self.tilesize dest_file = os.path.join(self.tmpdir, "djangowarpedraster" + str(zoom) + ".tif") snapped_dataset = self.dataset.warp( { "name": dest_file, "origin": [bounds[0], bounds[3]], "scale": [tilescale, -tilescale], "width": sizex, "height": sizey, } ) self.log("Creating {0} tiles for zoom {1}.".format(nr_of_tiles, zoom)) counter = 0 for tilex in range(indexrange[0], indexrange[2] + 1): for tiley in range(indexrange[1], indexrange[3] + 1): # Log progress counter += 1 if counter % 250 == 0: self.log("{0} tiles created at zoom {1}".format(counter, zoom)) # Calculate raster tile origin bounds = tiler.tile_bounds(tilex, tiley, zoom) # Construct band data arrays pixeloffset = ((tilex - indexrange[0]) * self.tilesize, (tiley - indexrange[1]) * self.tilesize) band_data = [ { "data": band.data(offset=pixeloffset, size=(self.tilesize, self.tilesize)), "nodata_value": band.nodata_value, } for band in snapped_dataset.bands ] # Add tile data to histogram if zoom == self.max_zoom: self.push_histogram(band_data) # Warp source raster into this tile (in memory) dest = GDALRaster( { "width": self.tilesize, "height": self.tilesize, "origin": [bounds[0], bounds[3]], "scale": [tilescale, -tilescale], "srid": WEB_MERCATOR_SRID, "datatype": snapped_dataset.bands[0].datatype(), "bands": band_data, } ) # Store tile RasterTile.objects.create(rast=dest, rasterlayer=self.rasterlayer, tilex=tilex, tiley=tiley, tilez=zoom) # Store histogram data if zoom == self.max_zoom: bandmetas = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer) for bandmeta in bandmetas: bandmeta.hist_values = self.hist_values[bandmeta.band].tolist() bandmeta.save() # Remove snapped dataset self.log("Removing snapped dataset.", zoom=zoom) snapped_dataset = None os.remove(dest_file) def push_histogram(self, data): """ Add data to band level histogram histogram. """ # Loop through bands of this tile for i, dat in enumerate(data): # Create histogram for new data with the same bins new_hist = numpy.histogram(dat["data"], bins=self.hist_bins[i]) # Add counts of this tile to band metadata histogram self.hist_values[i] += new_hist[0] def drop_empty_rasters(self): """ Remove rasters that are only no-data from the current rasterlayer. """ self.log("Dropping empty raster tiles.", status=self.rasterlayer.parsestatus.DROPPING_EMPTY_TILES) # Setup SQL command sql = ("DELETE FROM raster_rastertile " "WHERE ST_Count(rast)=0 " "AND rasterlayer_id={0}").format( self.rasterlayer.id ) # Run SQL to drop empty tiles cursor = connection.cursor() cursor.execute(sql) def parse_raster_layer(self): """ This function pushes the raster data from the Raster Layer into the RasterTile table. """ try: # Clean previous parse log self.log("Started parsing raster file", reset=True, status=self.rasterlayer.parsestatus.DOWNLOADING_FILE) # Download, unzip and open raster file self.get_raster_file() self.open_raster_file() # Remove existing tiles for this layer before loading new ones self.rasterlayer.rastertile_set.all().delete() # Transform raster to global srid if self.dataset.srs.srid == WEB_MERCATOR_SRID: self.log("Dataset already in SRID {0}, skipping transform".format(WEB_MERCATOR_SRID)) else: self.log( "Transforming raster to SRID {0}".format(WEB_MERCATOR_SRID), status=self.rasterlayer.parsestatus.REPROJECTING_RASTER, ) self.dataset = self.dataset.transform(WEB_MERCATOR_SRID) # Compute max zoom at the web mercator projection self.max_zoom = tiler.closest_zoomlevel(abs(self.dataset.scale.x)) # Store max zoom level in metadata self.rasterlayer.metadata.max_zoom = self.max_zoom self.rasterlayer.metadata.save() # Reduce max zoom by one if zoomdown flag was disabled if not self.zoomdown: self.max_zoom -= 1 self.log("Started creating tiles", status=self.rasterlayer.parsestatus.CREATING_TILES) # Loop through all lower zoom levels and create tiles to # setup TMS aligned tiles in world mercator for iz in range(self.max_zoom + 1): self.create_tiles(iz) self.drop_empty_rasters() # Send signal for end of parsing rasterlayers_parser_ended.send(sender=self.rasterlayer.__class__, instance=self.rasterlayer) # Log success of parsing self.log("Successfully finished parsing raster", status=self.rasterlayer.parsestatus.FINISHED) except: self.log(traceback.format_exc(), status=self.rasterlayer.parsestatus.FAILED) raise finally: self.close_raster_file() shutil.rmtree(self.tmpdir)
class RasterLayerParser(object): """ Class to parse raster layers. """ def __init__(self, rasterlayer_id): self.rasterlayer = RasterLayer.objects.get(id=rasterlayer_id) # Set raster tilesize self.tilesize = int(getattr(settings, 'RASTER_TILESIZE', WEB_MERCATOR_TILESIZE)) def log(self, msg, status=None, zoom=None): """ Write a message to the parse log of the rasterlayer instance and update the parse status object. """ parsestatus = self.rasterlayer.parsestatus parsestatus.refresh_from_db() if status is not None: parsestatus.status = status if zoom is not None and zoom not in parsestatus.tile_levels: parsestatus.tile_levels.append(zoom) parsestatus.tile_levels.sort() # Prepare datetime stamp for log now = '[{0}] '.format(datetime.datetime.now().strftime('%Y-%m-%d %T')) if parsestatus.log: now = '\n' + now parsestatus.log += now + msg parsestatus.save() def open_raster_file(self): """ Get raster source file to extract tiles from. This makes a local copy of rasterfile, unzips the raster and reprojects it into web mercator if necessary. The reprojected raster is stored for reuse such that reprojection does only happen once. The local copy of the raster is needed if files are stored on remote storages. """ reproj, created = RasterLayerReprojected.objects.get_or_create(rasterlayer=self.rasterlayer) # Check if the raster has already been reprojected has_reprojected = reproj.rasterfile.name not in (None, '') # Create workdir raster_workdir = getattr(settings, 'RASTER_WORKDIR', None) self.tmpdir = tempfile.mkdtemp(dir=raster_workdir) # Choose source for raster data, use the reprojected version if it exists. if self.rasterlayer.source_url and not has_reprojected: url_path = urlparse(self.rasterlayer.source_url).path filename = url_path.split('/')[-1] filepath = os.path.join(self.tmpdir, filename) urlretrieve(self.rasterlayer.source_url, filepath) else: if has_reprojected: rasterfile_source = reproj.rasterfile else: rasterfile_source = self.rasterlayer.rasterfile if not rasterfile_source.name: raise RasterException('No data source found. Provide a rasterfile or a source url.') # Copy raster file source to local folder filepath = os.path.join(self.tmpdir, os.path.basename(rasterfile_source.name)) rasterfile = open(filepath, 'wb') for chunk in rasterfile_source.chunks(): rasterfile.write(chunk) rasterfile.close() # If the raster file is compressed, decompress it, otherwise try to # open the source file directly. if os.path.splitext(filepath)[1].lower() == '.zip': # Open and extract zipfile zf = zipfile.ZipFile(filepath) zf.extractall(self.tmpdir) # Remove zipfile os.remove(filepath) # Get filelist from directory matches = [] for root, dirnames, filenames in os.walk(self.tmpdir): for filename in fnmatch.filter(filenames, '*.*'): matches.append(os.path.join(root, filename)) # Open the first raster file found in the matched files. self.dataset = None for match in matches: try: self.dataset = GDALRaster(match) break except GDALException: pass # Raise exception if no file could be opened by gdal. if not self.dataset: raise RasterException('Could not open rasterfile.') else: self.dataset = GDALRaster(filepath) # Override srid if provided if self.rasterlayer.srid: try: self.dataset = GDALRaster(self.dataset.name, write=True) except GDALException: raise RasterException( 'Could not override srid because the driver for this ' 'type of raster does not support write mode.' ) self.dataset.srs = self.rasterlayer.srid def reproject_rasterfile(self): """ Reproject the rasterfile into web mercator. """ # Return if reprojected rasterfile already exists. if hasattr(self.rasterlayer, 'reprojected') and self.rasterlayer.reprojected.rasterfile.name: return # Return if the raster already has the right projection # and nodata value is acceptable. if self.dataset.srs.srid == WEB_MERCATOR_SRID: # SRID was not manually specified. if self.rasterlayer.nodata in ('', None): return # All bands from dataset already have the same nodata value as the # one that was manually specified. if all([self.rasterlayer.nodata == band.nodata_value for band in self.dataset.bands]): return else: # Log projection change if original raster is not in web mercator. self.log( 'Transforming raster to SRID {0}'.format(WEB_MERCATOR_SRID), status=self.rasterlayer.parsestatus.REPROJECTING_RASTER, ) # Reproject the dataset. self.dataset = self.dataset.transform( WEB_MERCATOR_SRID, driver=INTERMEDIATE_RASTER_FORMAT, ) # Manually override nodata value if neccessary if self.rasterlayer.nodata not in ('', None): self.log( 'Setting no data values to {0}.'.format(self.rasterlayer.nodata), status=self.rasterlayer.parsestatus.REPROJECTING_RASTER, ) for band in self.dataset.bands: band.nodata_value = float(self.rasterlayer.nodata) # Compress reprojected raster file and store it if self.rasterlayer.store_reprojected: dest = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.zip') dest_zip = zipfile.ZipFile(dest.name, 'w', allowZip64=True) dest_zip.write( filename=self.dataset.name, arcname=os.path.basename(self.dataset.name), compress_type=zipfile.ZIP_DEFLATED, ) dest_zip.close() # Store zip file in reprojected raster model self.rasterlayer.reprojected.rasterfile = File(open(dest_zip.filename, 'rb')) self.rasterlayer.reprojected.save() self.log('Finished transforming raster.') def create_initial_histogram_buckets(self): """ Gets the empty histogram arrays for statistics collection. """ self.hist_values = [] self.hist_bins = [] for i, band in enumerate(self.dataset.bands): bandmeta = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer, band=i).first() self.hist_values.append(numpy.array(bandmeta.hist_values)) self.hist_bins.append(numpy.array(bandmeta.hist_bins)) def extract_metadata(self): """ Extract and store metadata for the raster and its bands. """ self.log('Extracting metadata from raster.') # Try to compute max zoom try: max_zoom = self.compute_max_zoom() except GDALException: raise RasterException('Failed to compute max zoom. Check the SRID of the raster.') # Extract global raster metadata meta = self.rasterlayer.metadata meta.uperleftx = self.dataset.origin.x meta.uperlefty = self.dataset.origin.y meta.width = self.dataset.width meta.height = self.dataset.height meta.scalex = self.dataset.scale.x meta.scaley = self.dataset.scale.y meta.skewx = self.dataset.skew.x meta.skewy = self.dataset.skew.y meta.numbands = len(self.dataset.bands) meta.srs_wkt = self.dataset.srs.wkt meta.srid = self.dataset.srs.srid meta.max_zoom = max_zoom meta.save() # Extract band metadata for i, band in enumerate(self.dataset.bands): bandmeta = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer, band=i).first() if not bandmeta: bandmeta = RasterLayerBandMetadata(rasterlayer=self.rasterlayer, band=i) bandmeta.nodata_value = band.nodata_value bandmeta.min = band.min bandmeta.max = band.max # Depending on Django version, the band statistics include std and mean. if hasattr(band, 'std'): bandmeta.std = band.std if hasattr(band, 'mean'): bandmeta.mean = band.mean bandmeta.save() self.log('Finished extracting metadata from raster.') def create_tiles(self, zoom_levels): """ Create tiles for input zoom levels, either a list or an integer. """ if isinstance(zoom_levels, int): self.populate_tile_level(zoom_levels) else: for zoom in zoom_levels: self.populate_tile_level(zoom) def populate_tile_level(self, zoom): """ Create tiles for this raster at the given zoomlevel. This routine first snaps the raster to the grid of the zoomlevel, then creates the tiles from the snapped raster. """ # Abort if zoom level is above resolution of the raster layer if zoom > self.max_zoom: return elif zoom == self.max_zoom: self.create_initial_histogram_buckets() # Compute the tile x-y-z index range for the rasterlayer for this zoomlevel bbox = self.dataset.extent quadrants = utils.quadrants(bbox, zoom) self.log('Creating {0} tiles in {1} quadrants at zoom {2}.'.format(self.nr_of_tiles(zoom), len(quadrants), zoom)) # Process quadrants in parallell quadrant_task_group = group(self.process_quadrant.si(indexrange, zoom) for indexrange in quadrants) quadrant_task_group.apply() # Store histogram data if zoom == self.max_zoom: bandmetas = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer) for bandmeta in bandmetas: bandmeta.hist_values = self.hist_values[bandmeta.band].tolist() bandmeta.save() self.log('Finished parsing at zoom level {0}.'.format(zoom), zoom=zoom) _quadrant_count = 0 @current_app.task(filter=task_method) def process_quadrant(self, indexrange, zoom): """ Create raster tiles for a quadrant of tiles defined by a x-y-z index range and a zoom level. """ self._quadrant_count += 1 self.log( 'Starting tile creation for quadrant {0} at zoom level {1}'.format(self._quadrant_count, zoom), status=self.rasterlayer.parsestatus.CREATING_TILES ) # Compute scale of tiles for this zoomlevel tilescale = utils.tile_scale(zoom) # Compute quadrant bounds and create destination file bounds = utils.tile_bounds(indexrange[0], indexrange[1], zoom) dest_file = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.tif') # Snap dataset to the quadrant snapped_dataset = self.dataset.warp({ 'name': dest_file.name, 'origin': [bounds[0], bounds[3]], 'scale': [tilescale, -tilescale], 'width': (indexrange[2] - indexrange[0] + 1) * self.tilesize, 'height': (indexrange[3] - indexrange[1] + 1) * self.tilesize, }) # Create all tiles in this quadrant in batches batch = [] for tilex in range(indexrange[0], indexrange[2] + 1): for tiley in range(indexrange[1], indexrange[3] + 1): # Calculate raster tile origin bounds = utils.tile_bounds(tilex, tiley, zoom) # Construct band data arrays pixeloffset = ( (tilex - indexrange[0]) * self.tilesize, (tiley - indexrange[1]) * self.tilesize ) band_data = [ { 'data': band.data(offset=pixeloffset, size=(self.tilesize, self.tilesize)), 'nodata_value': band.nodata_value } for band in snapped_dataset.bands ] # Ignore tile if its only nodata. if all([numpy.all(dat['data'] == dat['nodata_value']) for dat in band_data]): continue # Add tile data to histogram if zoom == self.max_zoom: self.push_histogram(band_data) # Warp source raster into this tile (in memory) dest = GDALRaster({ 'width': self.tilesize, 'height': self.tilesize, 'origin': [bounds[0], bounds[3]], 'scale': [tilescale, -tilescale], 'srid': WEB_MERCATOR_SRID, 'datatype': snapped_dataset.bands[0].datatype(), 'bands': band_data, }) # Store tile in batch array batch.append( RasterTile( rast=dest, rasterlayer_id=self.rasterlayer.id, tilex=tilex, tiley=tiley, tilez=zoom ) ) # Commit batch to database and reset it if len(batch) == BATCH_STEP_SIZE: RasterTile.objects.bulk_create(batch) batch = [] # Commit remaining objects if len(batch): RasterTile.objects.bulk_create(batch) def push_histogram(self, data): """ Add data to band level histogram. """ # Loop through bands of this tile for i, dat in enumerate(data): # Create histogram for new data with the same bins new_hist = numpy.histogram(dat['data'], bins=self.hist_bins[i]) # Add counts of this tile to band metadata histogram self.hist_values[i] += new_hist[0] def drop_all_tiles(self): """ Delete all existing tiles for this parser's rasterlayer. """ self.log('Clearing all existing tiles.') self.rasterlayer.rastertile_set.all().delete() self.log('Finished clearing existing tiles.') def send_success_signal(self): """ Send parser end signal for other dependencies to be handling new tiles. """ self.log( 'Successfully finished parsing raster', status=self.rasterlayer.parsestatus.FINISHED ) rasterlayers_parser_ended.send(sender=self.rasterlayer.__class__, instance=self.rasterlayer) def compute_max_zoom(self): """ Set max zoom property based on rasterlayer metadata. """ # Return manual override value if provided if self.rasterlayer.max_zoom is not None: return self.rasterlayer.max_zoom if self.dataset.srs.srid == WEB_MERCATOR_SRID: # For rasters in web mercator, use the scale directly scale = abs(self.dataset.scale.x) else: # Create a line from the center of the raster to a point that is # one pixel width away from the center. xcenter = self.dataset.extent[0] + (self.dataset.extent[2] - self.dataset.extent[0]) / 2 ycenter = self.dataset.extent[1] + (self.dataset.extent[3] - self.dataset.extent[1]) / 2 linestring = 'LINESTRING({} {}, {} {})'.format( xcenter, ycenter, xcenter + self.dataset.scale.x, ycenter ) line = OGRGeometry(linestring, srs=self.dataset.srs) # Tansform the line into web mercator. line.transform(WEB_MERCATOR_SRID) # Use the lenght of the transformed line as scale. scale = line.geos.length return utils.closest_zoomlevel(scale) @property def max_zoom(self): # Return manual override value if provided if self.rasterlayer.max_zoom is not None: return self.rasterlayer.max_zoom # Get max zoom from metadata if not hasattr(self.rasterlayer, 'metadata'): raise RasterException('Could not determine max zoom level.') max_zoom = self.rasterlayer.metadata.max_zoom # Reduce max zoom by one if zoomdown flag was disabled if not self.rasterlayer.next_higher: max_zoom -= 1 return max_zoom def nr_of_tiles(self, zoom): """ Compute the number of tiles for the rasterlayer on a given zoom level. """ bbox = self.dataset.extent indexrange = utils.tile_index_range(bbox, zoom) return (indexrange[2] - indexrange[0] + 1) * (indexrange[3] - indexrange[1] + 1)