Exemple #1
0
    def test_raster_metadata_property(self):
        data = self.rs.metadata
        self.assertEqual(data['DEFAULT'], {'AREA_OR_POINT': 'Area'})
        self.assertEqual(data['IMAGE_STRUCTURE'], {'INTERLEAVE': 'BAND'})

        # Create file-based raster from scratch
        source = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{'data': range(4), 'nodata_value': 99}],
        })
        # Set metadata on raster and on a band.
        metadata = {
            'DEFAULT': {'OWNER': 'Django', 'VERSION': '1.0', 'AREA_OR_POINT': 'Point'},
        }
        source.metadata = metadata
        source.bands[0].metadata = metadata
        self.assertEqual(source.metadata['DEFAULT'], metadata['DEFAULT'])
        self.assertEqual(source.bands[0].metadata['DEFAULT'], metadata['DEFAULT'])
        # Update metadata on raster.
        metadata = {
            'DEFAULT': {'VERSION': '2.0'},
        }
        source.metadata = metadata
        self.assertEqual(source.metadata['DEFAULT']['VERSION'], '2.0')
        # Remove metadata on raster.
        metadata = {
            'DEFAULT': {'OWNER': None},
        }
        source.metadata = metadata
        self.assertNotIn('OWNER', source.metadata['DEFAULT'])
Exemple #2
0
 def test_geotransform_and_friends(self):
     # Assert correct values for file based raster
     self.assertEqual(
         self.rs.geotransform,
         [511700.4680706557, 100.0, 0.0, 435103.3771231986, 0.0, -100.0]
     )
     self.assertEqual(self.rs.origin, [511700.4680706557, 435103.3771231986])
     self.assertEqual(self.rs.origin.x, 511700.4680706557)
     self.assertEqual(self.rs.origin.y, 435103.3771231986)
     self.assertEqual(self.rs.scale, [100.0, -100.0])
     self.assertEqual(self.rs.scale.x, 100.0)
     self.assertEqual(self.rs.scale.y, -100.0)
     self.assertEqual(self.rs.skew, [0, 0])
     self.assertEqual(self.rs.skew.x, 0)
     self.assertEqual(self.rs.skew.y, 0)
     # Create in-memory rasters and change gtvalues
     rsmem = GDALRaster(JSON_RASTER)
     rsmem.geotransform = range(6)
     self.assertEqual(rsmem.geotransform, [float(x) for x in range(6)])
     self.assertEqual(rsmem.origin, [0, 3])
     self.assertEqual(rsmem.origin.x, 0)
     self.assertEqual(rsmem.origin.y, 3)
     self.assertEqual(rsmem.scale, [1, 5])
     self.assertEqual(rsmem.scale.x, 1)
     self.assertEqual(rsmem.scale.y, 5)
     self.assertEqual(rsmem.skew, [2, 4])
     self.assertEqual(rsmem.skew.x, 2)
     self.assertEqual(rsmem.skew.y, 4)
     self.assertEqual(rsmem.width, 5)
     self.assertEqual(rsmem.height, 5)
Exemple #3
0
    def test_raster_transform(self):
        # Prepare tempfile and nodata value
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
        ndv = 99

        # Create in file based raster
        source = GDALRaster({
            'datatype': 1,
            'driver': 'tif',
            'name': rstfile.name,
            'width': 5,
            'height': 5,
            'nr_of_bands': 1,
            'srid': 4326,
            'origin': (-5, 5),
            'scale': (2, -2),
            'skew': (0, 0),
            'bands': [{
                'data': range(25),
                'nodata_value': ndv,
            }],
        })

        # Transform raster into srid 4326.
        target = source.transform(3086)

        # Reload data from disk
        target = GDALRaster(target.name)

        self.assertEqual(target.srs.srid, 3086)
        self.assertEqual(target.width, 7)
        self.assertEqual(target.height, 7)
        self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype())
        self.assertAlmostEqual(target.origin[0], 9124842.791079799)
        self.assertAlmostEqual(target.origin[1], 1589911.6476407414)
        self.assertAlmostEqual(target.scale[0], 223824.82664250192)
        self.assertAlmostEqual(target.scale[1], -223824.82664250192)
        self.assertEqual(target.skew, [0, 0])

        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # The reprojection of a raster that spans over a large area
        # skews the data matrix and might introduce nodata values.
        self.assertEqual(
            result,
            [
                ndv, ndv, ndv, ndv, 4, ndv, ndv,
                ndv, ndv, 2, 3, 9, ndv, ndv,
                ndv, 1, 2, 8, 13, 19, ndv,
                0, 6, 6, 12, 18, 18, 24,
                ndv, 10, 11, 16, 22, 23, ndv,
                ndv, ndv, 15, 21, 22, ndv, ndv,
                ndv, ndv, 20, ndv, ndv, ndv, ndv,
            ]
        )
Exemple #4
0
 def test_rs_srid(self):
     rast = GDALRaster({
         'width': 16,
         'height': 16,
         'srid': 4326,
     })
     self.assertEqual(rast.srid, 4326)
     rast.srid = 3086
     self.assertEqual(rast.srid, 3086)
Exemple #5
0
    def test_raster_warp(self):
        # Create in memory raster
        source = GDALRaster({
            'datatype': 1,
            'driver': 'MEM',
            'name': 'sourceraster',
            'width': 4,
            'height': 4,
            'nr_of_bands': 1,
            'srid': 3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(16),
                'nodata_value': 255,
            }],
        })

        # Test altering the scale, width, and height of a raster
        data = {
            'scale': [200, -200],
            'width': 2,
            'height': 2,
        }
        target = source.warp(data)
        self.assertEqual(target.width, data['width'])
        self.assertEqual(target.height, data['height'])
        self.assertEqual(target.scale, data['scale'])
        self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype())
        self.assertEqual(target.name, 'sourceraster_copy.MEM')
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(result, [5, 7, 13, 15])

        # Test altering the name and datatype (to float)
        data = {
            'name': '/path/to/targetraster.tif',
            'datatype': 6,
        }
        target = source.warp(data)
        self.assertEqual(target.bands[0].datatype(), 6)
        self.assertEqual(target.name, '/path/to/targetraster.tif')
        self.assertEqual(target.driver.name, 'MEM')
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(
            result,
            [0.0, 1.0, 2.0, 3.0,
             4.0, 5.0, 6.0, 7.0,
             8.0, 9.0, 10.0, 11.0,
             12.0, 13.0, 14.0, 15.0]
        )
Exemple #6
0
 def test_geotransform_bad_inputs(self):
     rsmem = GDALRaster(JSON_RASTER)
     error_geotransforms = [
         [1, 2],
         [1, 2, 3, 4, 5, 'foo'],
         [1, 2, 3, 4, 5, 6, 'foo'],
     ]
     msg = 'Geotransform must consist of 6 numeric values.'
     for geotransform in error_geotransforms:
         with self.subTest(i=geotransform), self.assertRaisesMessage(ValueError, msg):
             rsmem.geotransform = geotransform
Exemple #7
0
    def test_db(self):
        # Make sure we can use PostGIS raster type
        raster = GDALRaster(os.path.realpath(os.path.join("app", "fixtures", "orthophoto.tif")), write=True)

        self.assertTrue(raster.srid == 32615)
        with transaction.atomic():
            # We cannot store offdb references with SRID different than the one declared (4326)
            self.assertRaises(InternalError, Task.objects.create, project=Project.objects.latest("created_at"),
                                       orthophoto=raster)

        # All OK when we transform to 4326
        task = Task.objects.create(project=Project.objects.latest("created_at"),
                                   orthophoto=raster.transform(4326))
        task.refresh_from_db()
        self.assertTrue(task.orthophoto.srid == 4326)
        self.assertTrue(task.orthophoto.width == 252) # not original size, warp happened
Exemple #8
0
    def test_band_data_replication(self):
        band = GDALRaster({
            'srid': 4326,
            'width': 3,
            'height': 3,
            'bands': [{'data': range(10, 19), 'nodata_value': 0}],
        }).bands[0]

        # Variations for input (data, shape, expected result).
        combos = (
            ([1], (1, 1), [1] * 9),
            (range(3), (1, 3), [0, 0, 0, 1, 1, 1, 2, 2, 2]),
            (range(3), (3, 1), [0, 1, 2, 0, 1, 2, 0, 1, 2]),
        )
        for combo in combos:
            band.data(combo[0], shape=combo[1])
            if numpy:
                numpy.testing.assert_equal(band.data(), numpy.array(combo[2]).reshape(3, 3))
            else:
                self.assertEqual(band.data(), list(combo[2]))
Exemple #9
0
 def test_vsi_buffer_property(self):
     # Create a vsi-based raster from scratch.
     rast = GDALRaster({
         'name': '/vsimem/raster.tif',
         'driver': 'tif',
         'width': 4,
         'height': 4,
         'srid': 4326,
         'bands': [{
             'data': range(16),
         }],
     })
     # Do a round trip from raster to buffer to raster.
     result = GDALRaster(rast.vsi_buffer).bands[0].data()
     if numpy:
         result = result.flatten().tolist()
     # Band data is equal to nodata value except on input block of ones.
     self.assertEqual(result, list(range(16)))
     # The vsi buffer is None for rasters that are not vsi based.
     self.assertIsNone(self.rs.vsi_buffer)
Exemple #10
0
    def test_raster_metadata_property(self):
        # Check for required gdal version.
        if GDAL_VERSION < (1, 11):
            msg = 'GDAL ≥ 1.11 is required for using the metadata property.'
            with self.assertRaisesMessage(ValueError, msg):
                self.rs.metadata
            return

        self.assertEqual(
            self.rs.metadata,
            {'DEFAULT': {'AREA_OR_POINT': 'Area'}, 'IMAGE_STRUCTURE': {'INTERLEAVE': 'BAND'}},
        )

        # Create file-based raster from scratch
        source = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{'data': range(4), 'nodata_value': 99}],
        })
        # Set metadata on raster and on a band.
        metadata = {
            'DEFAULT': {'OWNER': 'Django', 'VERSION': '1.0', 'AREA_OR_POINT': 'Point', },
        }
        source.metadata = metadata
        source.bands[0].metadata = metadata
        self.assertEqual(source.metadata['DEFAULT'], metadata['DEFAULT'])
        self.assertEqual(source.bands[0].metadata['DEFAULT'], metadata['DEFAULT'])
        # Update metadata on raster.
        metadata = {
            'DEFAULT': {'VERSION': '2.0', },
        }
        source.metadata = metadata
        self.assertEqual(source.metadata['DEFAULT']['VERSION'], '2.0')
        # Remove metadata on raster.
        metadata = {
            'DEFAULT': {'OWNER': None, },
        }
        source.metadata = metadata
        self.assertNotIn('OWNER', source.metadata['DEFAULT'])
Exemple #11
0
    def test_raster_warp(self):
        # Create in memory raster
        source = GDALRaster(
            {
                "datatype": 1,
                "driver": "MEM",
                "name": "sourceraster",
                "width": 4,
                "height": 4,
                "nr_of_bands": 1,
                "srid": 3086,
                "origin": (500000, 400000),
                "scale": (100, -100),
                "skew": (0, 0),
                "bands": [{"data": range(16), "nodata_value": 255}],
            }
        )

        # Test altering the scale, width, and height of a raster
        data = {"scale": [200, -200], "width": 2, "height": 2}
        target = source.warp(data)
        self.assertEqual(target.width, data["width"])
        self.assertEqual(target.height, data["height"])
        self.assertEqual(target.scale, data["scale"])
        self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype())
        self.assertEqual(target.name, "sourceraster_copy.MEM")
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(result, [5, 7, 13, 15])

        # Test altering the name and datatype (to float)
        data = {"name": "/path/to/targetraster.tif", "datatype": 6}
        target = source.warp(data)
        self.assertEqual(target.bands[0].datatype(), 6)
        self.assertEqual(target.name, "/path/to/targetraster.tif")
        self.assertEqual(target.driver.name, "MEM")
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(result, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0])
Exemple #12
0
 def test_raster_warp_nodata_zone(self):
     # Create in memory raster.
     source = GDALRaster({
         'datatype': 1,
         'driver': 'MEM',
         'width': 4,
         'height': 4,
         'srid': 3086,
         'origin': (500000, 400000),
         'scale': (100, -100),
         'skew': (0, 0),
         'bands': [{
             'data': range(16),
             'nodata_value': 23,
         }],
     })
     # Warp raster onto a location that does not cover any pixels of the original.
     result = source.warp({'origin': (200000, 200000)}).bands[0].data()
     if numpy:
         result = result.flatten().tolist()
     # The result is an empty raster filled with the correct nodata value.
     self.assertEqual(result, [23] * 16)
Exemple #13
0
    def open_raster_file(self):
        """
        Open the raster file as GDALRaster and set nodata-values.
        """
        self.log("Opening raster file as GDALRaster.")

        # Open raster file
        self.dataset = GDALRaster(os.path.join(self.tmpdir, self.rastername), write=True)

        # Make sure nodata value is set from input
        self.hist_values = []
        self.hist_bins = []
        for i, band in enumerate(self.dataset.bands):
            if self.rasterlayer.nodata is not None:
                band.nodata_value = float(self.rasterlayer.nodata)

            # Create band metatdata object
            bandmeta = RasterLayerBandMetadata.objects.create(
                rasterlayer=self.rasterlayer, band=i, nodata_value=band.nodata_value, min=band.min, max=band.max
            )

            # Prepare numpy hist values and bins
            self.hist_values.append(numpy.array(bandmeta.hist_values))
            self.hist_bins.append(numpy.array(bandmeta.hist_bins))

        # Store original metadata for this raster
        meta = self.rasterlayer.metadata

        meta.uperleftx = self.dataset.origin.x
        meta.uperlefty = self.dataset.origin.y
        meta.width = self.dataset.width
        meta.height = self.dataset.height
        meta.scalex = self.dataset.scale.x
        meta.scaley = self.dataset.scale.y
        meta.skewx = self.dataset.skew.x
        meta.skewy = self.dataset.skew.y
        meta.numbands = len(self.dataset.bands)
        meta.srs_wkt = self.dataset.srs.wkt
        meta.srid = self.dataset.srs.srid

        meta.save()
Exemple #14
0
    def test_get_pixel_value(self):
        raster = GDALRaster({
            'width': 5,
            'height': 5,
            'srid': 4326,
            'bands': [{
                'data': range(25)
            }],
            'origin': (2, 2),
            'scale': (1, 1)
        })

        # Pixel value at origin.
        point = OGRGeometry('SRID=4326;POINT(2 2)')
        result = pixel_value_from_point(raster, point)
        self.assertEqual(result, 0)

        # Coords as tuple.
        result = pixel_value_from_point(raster, (2, 2))
        self.assertEqual(result, 0)

        # Point in different projection.
        point = OGRGeometry(
            'SRID=3857;POINT(222638.9815865472 222684.20850554455)')
        result = pixel_value_from_point(raster, point)
        self.assertEqual(result, 0)

        # Pixel value outside of raster.
        point = OGRGeometry('SRID=4326;POINT(-2 2)')
        result = pixel_value_from_point(raster, point)
        self.assertEqual(result, None)

        point = OGRGeometry('SRID=4326;POINT(8 8)')
        result = pixel_value_from_point(raster, point)
        self.assertEqual(result, None)

        # Pixel values within the raster.
        point = OGRGeometry('SRID=4326;POINT(3.5 2)')
        result = pixel_value_from_point(raster, point)
        self.assertEqual(result, 1)

        point = OGRGeometry('SRID=4326;POINT(2 3.5)')
        result = pixel_value_from_point(raster, point)
        self.assertEqual(result, 5)

        point = OGRGeometry('SRID=4326;POINT(6.999 6.9999)')
        result = pixel_value_from_point(raster, point)
        self.assertEqual(result, 24)

        # Pixel value at "outer" edge of raster.
        point = OGRGeometry('SRID=4326;POINT(7 7)')
        result = pixel_value_from_point(raster, point)
        self.assertEqual(result, 24)

        # Point without srs specified.
        point = OGRGeometry('POINT(2 2)')
        with self.assertRaises(ValueError):
            pixel_value_from_point(raster, point)

        # Raster with negative scale on y axis.
        raster = GDALRaster({
            'width': 5,
            'height': 5,
            'srid': 4326,
            'bands': [{
                'data': range(25)
            }],
            'origin': (2, 2),
            'scale': (1, -1)
        })
        point = OGRGeometry('SRID=4326;POINT(3 1)')
        result = pixel_value_from_point(raster, point)
        self.assertEqual(result, 6)
Exemple #15
0
class GDALRasterTests(SimpleTestCase):
    """
    Test a GDALRaster instance created from a file (GeoTiff).
    """
    def setUp(self):
        self.rs_path = os.path.join(os.path.dirname(__file__), '../data/rasters/raster.tif')
        self.rs = GDALRaster(self.rs_path)

    def test_rs_name_repr(self):
        self.assertEqual(self.rs_path, self.rs.name)
        self.assertRegex(repr(self.rs), r"<Raster object at 0x\w+>")

    def test_rs_driver(self):
        self.assertEqual(self.rs.driver.name, 'GTiff')

    def test_rs_size(self):
        self.assertEqual(self.rs.width, 163)
        self.assertEqual(self.rs.height, 174)

    def test_rs_srs(self):
        self.assertEqual(self.rs.srs.srid, 3086)
        self.assertEqual(self.rs.srs.units, (1.0, 'metre'))

    def test_rs_srid(self):
        rast = GDALRaster({
            'width': 16,
            'height': 16,
            'srid': 4326,
        })
        self.assertEqual(rast.srid, 4326)
        rast.srid = 3086
        self.assertEqual(rast.srid, 3086)

    def test_geotransform_and_friends(self):
        # Assert correct values for file based raster
        self.assertEqual(
            self.rs.geotransform,
            [511700.4680706557, 100.0, 0.0, 435103.3771231986, 0.0, -100.0]
        )
        self.assertEqual(self.rs.origin, [511700.4680706557, 435103.3771231986])
        self.assertEqual(self.rs.origin.x, 511700.4680706557)
        self.assertEqual(self.rs.origin.y, 435103.3771231986)
        self.assertEqual(self.rs.scale, [100.0, -100.0])
        self.assertEqual(self.rs.scale.x, 100.0)
        self.assertEqual(self.rs.scale.y, -100.0)
        self.assertEqual(self.rs.skew, [0, 0])
        self.assertEqual(self.rs.skew.x, 0)
        self.assertEqual(self.rs.skew.y, 0)
        # Create in-memory rasters and change gtvalues
        rsmem = GDALRaster(JSON_RASTER)
        # geotransform accepts both floats and ints
        rsmem.geotransform = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
        self.assertEqual(rsmem.geotransform, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0])
        rsmem.geotransform = range(6)
        self.assertEqual(rsmem.geotransform, [float(x) for x in range(6)])
        self.assertEqual(rsmem.origin, [0, 3])
        self.assertEqual(rsmem.origin.x, 0)
        self.assertEqual(rsmem.origin.y, 3)
        self.assertEqual(rsmem.scale, [1, 5])
        self.assertEqual(rsmem.scale.x, 1)
        self.assertEqual(rsmem.scale.y, 5)
        self.assertEqual(rsmem.skew, [2, 4])
        self.assertEqual(rsmem.skew.x, 2)
        self.assertEqual(rsmem.skew.y, 4)
        self.assertEqual(rsmem.width, 5)
        self.assertEqual(rsmem.height, 5)

    def test_geotransform_bad_inputs(self):
        rsmem = GDALRaster(JSON_RASTER)
        error_geotransforms = [
            [1, 2],
            [1, 2, 3, 4, 5, 'foo'],
            [1, 2, 3, 4, 5, 6, 'foo'],
        ]
        msg = 'Geotransform must consist of 6 numeric values.'
        for geotransform in error_geotransforms:
            with self.subTest(i=geotransform), self.assertRaisesMessage(ValueError, msg):
                rsmem.geotransform = geotransform

    def test_rs_extent(self):
        self.assertEqual(
            self.rs.extent,
            (511700.4680706557, 417703.3771231986, 528000.4680706557, 435103.3771231986)
        )

    def test_rs_bands(self):
        self.assertEqual(len(self.rs.bands), 1)
        self.assertIsInstance(self.rs.bands[0], GDALBand)

    def test_memory_based_raster_creation(self):
        # Create uint8 raster with full pixel data range (0-255)
        rast = GDALRaster({
            'datatype': 1,
            'width': 16,
            'height': 16,
            'srid': 4326,
            'bands': [{
                'data': range(256),
                'nodata_value': 255,
            }],
        })

        # Get array from raster
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # Assert data is same as original input
        self.assertEqual(result, list(range(256)))

    def test_file_based_raster_creation(self):
        # Prepare tempfile
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')

        # Create file-based raster from scratch
        GDALRaster({
            'datatype': self.rs.bands[0].datatype(),
            'driver': 'tif',
            'name': rstfile.name,
            'width': 163,
            'height': 174,
            'nr_of_bands': 1,
            'srid': self.rs.srs.wkt,
            'origin': (self.rs.origin.x, self.rs.origin.y),
            'scale': (self.rs.scale.x, self.rs.scale.y),
            'skew': (self.rs.skew.x, self.rs.skew.y),
            'bands': [{
                'data': self.rs.bands[0].data(),
                'nodata_value': self.rs.bands[0].nodata_value,
            }],
        })

        # Reload newly created raster from file
        restored_raster = GDALRaster(rstfile.name)
        # Presence of TOWGS84 depend on GDAL/Proj versions.
        self.assertEqual(
            restored_raster.srs.wkt.replace('TOWGS84[0,0,0,0,0,0,0],', ''),
            self.rs.srs.wkt.replace('TOWGS84[0,0,0,0,0,0,0],', '')
        )
        self.assertEqual(restored_raster.geotransform, self.rs.geotransform)
        if numpy:
            numpy.testing.assert_equal(
                restored_raster.bands[0].data(),
                self.rs.bands[0].data()
            )
        else:
            self.assertEqual(restored_raster.bands[0].data(), self.rs.bands[0].data())

    def test_nonexistent_file(self):
        msg = 'Unable to read raster source input "nonexistent.tif".'
        with self.assertRaisesMessage(GDALException, msg):
            GDALRaster('nonexistent.tif')

    def test_vsi_raster_creation(self):
        # Open a raster as a file object.
        with open(self.rs_path, 'rb') as dat:
            # Instantiate a raster from the file binary buffer.
            vsimem = GDALRaster(dat.read())
        # The data of the in-memory file is equal to the source file.
        result = vsimem.bands[0].data()
        target = self.rs.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
            target = target.flatten().tolist()
        self.assertEqual(result, target)

    def test_vsi_raster_deletion(self):
        path = '/vsimem/raster.tif'
        # Create a vsi-based raster from scratch.
        vsimem = GDALRaster({
            'name': path,
            'driver': 'tif',
            'width': 4,
            'height': 4,
            'srid': 4326,
            'bands': [{
                'data': range(16),
            }],
        })
        # The virtual file exists.
        rst = GDALRaster(path)
        self.assertEqual(rst.width, 4)
        # Delete GDALRaster.
        del vsimem
        del rst
        # The virtual file has been removed.
        msg = 'Could not open the datasource at "/vsimem/raster.tif"'
        with self.assertRaisesMessage(GDALException, msg):
            GDALRaster(path)

    def test_vsi_invalid_buffer_error(self):
        msg = 'Failed creating VSI raster from the input buffer.'
        with self.assertRaisesMessage(GDALException, msg):
            GDALRaster(b'not-a-raster-buffer')

    def test_vsi_buffer_property(self):
        # Create a vsi-based raster from scratch.
        rast = GDALRaster({
            'name': '/vsimem/raster.tif',
            'driver': 'tif',
            'width': 4,
            'height': 4,
            'srid': 4326,
            'bands': [{
                'data': range(16),
            }],
        })
        # Do a round trip from raster to buffer to raster.
        result = GDALRaster(rast.vsi_buffer).bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to nodata value except on input block of ones.
        self.assertEqual(result, list(range(16)))
        # The vsi buffer is None for rasters that are not vsi based.
        self.assertIsNone(self.rs.vsi_buffer)

    def test_offset_size_and_shape_on_raster_creation(self):
        rast = GDALRaster({
            'datatype': 1,
            'width': 4,
            'height': 4,
            'srid': 4326,
            'bands': [{
                'data': (1,),
                'offset': (1, 1),
                'size': (2, 2),
                'shape': (1, 1),
                'nodata_value': 2,
            }],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to nodata value except on input block of ones.
        self.assertEqual(
            result,
            [2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2]
        )

    def test_set_nodata_value_on_raster_creation(self):
        # Create raster filled with nodata values.
        rast = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{'nodata_value': 23}],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # All band data is equal to nodata value.
        self.assertEqual(result, [23] * 4)

    def test_set_nodata_none_on_raster_creation(self):
        # Create raster without data and without nodata value.
        rast = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{'nodata_value': None}],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to zero because no nodata value has been specified.
        self.assertEqual(result, [0] * 4)

    def test_raster_metadata_property(self):
        data = self.rs.metadata
        self.assertEqual(data['DEFAULT'], {'AREA_OR_POINT': 'Area'})
        self.assertEqual(data['IMAGE_STRUCTURE'], {'INTERLEAVE': 'BAND'})

        # Create file-based raster from scratch
        source = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{'data': range(4), 'nodata_value': 99}],
        })
        # Set metadata on raster and on a band.
        metadata = {
            'DEFAULT': {'OWNER': 'Django', 'VERSION': '1.0', 'AREA_OR_POINT': 'Point'},
        }
        source.metadata = metadata
        source.bands[0].metadata = metadata
        self.assertEqual(source.metadata['DEFAULT'], metadata['DEFAULT'])
        self.assertEqual(source.bands[0].metadata['DEFAULT'], metadata['DEFAULT'])
        # Update metadata on raster.
        metadata = {
            'DEFAULT': {'VERSION': '2.0'},
        }
        source.metadata = metadata
        self.assertEqual(source.metadata['DEFAULT']['VERSION'], '2.0')
        # Remove metadata on raster.
        metadata = {
            'DEFAULT': {'OWNER': None},
        }
        source.metadata = metadata
        self.assertNotIn('OWNER', source.metadata['DEFAULT'])

    def test_raster_info_accessor(self):
        infos = self.rs.info
        # Data
        info_lines = [line.strip() for line in infos.split('\n') if line.strip() != '']
        for line in [
            'Driver: GTiff/GeoTIFF',
            'Files: {}'.format(self.rs_path),
            'Size is 163, 174',
            'Origin = (511700.468070655711927,435103.377123198588379)',
            'Pixel Size = (100.000000000000000,-100.000000000000000)',
            'Metadata:',
            'AREA_OR_POINT=Area',
            'Image Structure Metadata:',
            'INTERLEAVE=BAND',
            'Band 1 Block=163x50 Type=Byte, ColorInterp=Gray',
            'NoData Value=15'
        ]:
            self.assertIn(line, info_lines)
        for line in [
            r'Upper Left  \(  511700.468,  435103.377\) \( 82d51\'46.1\d"W, 27d55\' 1.5\d"N\)',
            r'Lower Left  \(  511700.468,  417703.377\) \( 82d51\'52.0\d"W, 27d45\'37.5\d"N\)',
            r'Upper Right \(  528000.468,  435103.377\) \( 82d41\'48.8\d"W, 27d54\'56.3\d"N\)',
            r'Lower Right \(  528000.468,  417703.377\) \( 82d41\'55.5\d"W, 27d45\'32.2\d"N\)',
            r'Center      \(  519850.468,  426403.377\) \( 82d46\'50.6\d"W, 27d50\'16.9\d"N\)',
        ]:
            self.assertRegex(infos, line)
        # CRS (skip the name because string depends on the GDAL/Proj versions).
        self.assertIn("NAD83 / Florida GDL Albers", infos)

    def test_compressed_file_based_raster_creation(self):
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
        # Make a compressed copy of an existing raster.
        compressed = self.rs.warp({'papsz_options': {'compress': 'packbits'}, 'name': rstfile.name})
        # Check physically if compression worked.
        self.assertLess(os.path.getsize(compressed.name), os.path.getsize(self.rs.name))
        # Create file-based raster with options from scratch.
        compressed = GDALRaster({
            'datatype': 1,
            'driver': 'tif',
            'name': rstfile.name,
            'width': 40,
            'height': 40,
            'srid': 3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(40 ^ 2),
                'nodata_value': 255,
            }],
            'papsz_options': {
                'compress': 'packbits',
                'pixeltype': 'signedbyte',
                'blockxsize': 23,
                'blockysize': 23,
            }
        })
        # Check if options used on creation are stored in metadata.
        # Reopening the raster ensures that all metadata has been written
        # to the file.
        compressed = GDALRaster(compressed.name)
        self.assertEqual(compressed.metadata['IMAGE_STRUCTURE']['COMPRESSION'], 'PACKBITS',)
        self.assertEqual(compressed.bands[0].metadata['IMAGE_STRUCTURE']['PIXELTYPE'], 'SIGNEDBYTE')
        self.assertIn('Block=40x23', compressed.info)

    def test_raster_warp(self):
        # Create in memory raster
        source = GDALRaster({
            'datatype': 1,
            'driver': 'MEM',
            'name': 'sourceraster',
            'width': 4,
            'height': 4,
            'nr_of_bands': 1,
            'srid': 3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(16),
                'nodata_value': 255,
            }],
        })

        # Test altering the scale, width, and height of a raster
        data = {
            'scale': [200, -200],
            'width': 2,
            'height': 2,
        }
        target = source.warp(data)
        self.assertEqual(target.width, data['width'])
        self.assertEqual(target.height, data['height'])
        self.assertEqual(target.scale, data['scale'])
        self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype())
        self.assertEqual(target.name, 'sourceraster_copy.MEM')
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(result, [5, 7, 13, 15])

        # Test altering the name and datatype (to float)
        data = {
            'name': '/path/to/targetraster.tif',
            'datatype': 6,
        }
        target = source.warp(data)
        self.assertEqual(target.bands[0].datatype(), 6)
        self.assertEqual(target.name, '/path/to/targetraster.tif')
        self.assertEqual(target.driver.name, 'MEM')
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(
            result,
            [0.0, 1.0, 2.0, 3.0,
             4.0, 5.0, 6.0, 7.0,
             8.0, 9.0, 10.0, 11.0,
             12.0, 13.0, 14.0, 15.0]
        )

    def test_raster_warp_nodata_zone(self):
        # Create in memory raster.
        source = GDALRaster({
            'datatype': 1,
            'driver': 'MEM',
            'width': 4,
            'height': 4,
            'srid': 3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(16),
                'nodata_value': 23,
            }],
        })
        # Warp raster onto a location that does not cover any pixels of the original.
        result = source.warp({'origin': (200000, 200000)}).bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # The result is an empty raster filled with the correct nodata value.
        self.assertEqual(result, [23] * 16)

    def test_raster_clone(self):
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
        tests = [
            ('MEM', '', 23),  # In memory raster.
            ('tif', rstfile.name, 99),  # In file based raster.
        ]
        for driver, name, nodata_value in tests:
            with self.subTest(driver=driver):
                source = GDALRaster({
                    'datatype': 1,
                    'driver': driver,
                    'name': name,
                    'width': 4,
                    'height': 4,
                    'srid': 3086,
                    'origin': (500000, 400000),
                    'scale': (100, -100),
                    'skew': (0, 0),
                    'bands': [{
                        'data': range(16),
                        'nodata_value': nodata_value,
                    }],
                })
                clone = source.clone()
                self.assertNotEqual(clone.name, source.name)
                self.assertEqual(clone._write, source._write)
                self.assertEqual(clone.srs.srid, source.srs.srid)
                self.assertEqual(clone.width, source.width)
                self.assertEqual(clone.height, source.height)
                self.assertEqual(clone.origin, source.origin)
                self.assertEqual(clone.scale, source.scale)
                self.assertEqual(clone.skew, source.skew)
                self.assertIsNot(clone, source)

    def test_raster_transform(self):
        tests = [
            3086,
            '3086',
            SpatialReference(3086),
        ]
        for srs in tests:
            with self.subTest(srs=srs):
                # Prepare tempfile and nodata value.
                rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
                ndv = 99
                # Create in file based raster.
                source = GDALRaster({
                    'datatype': 1,
                    'driver': 'tif',
                    'name': rstfile.name,
                    'width': 5,
                    'height': 5,
                    'nr_of_bands': 1,
                    'srid': 4326,
                    'origin': (-5, 5),
                    'scale': (2, -2),
                    'skew': (0, 0),
                    'bands': [{
                        'data': range(25),
                        'nodata_value': ndv,
                    }],
                })

                target = source.transform(srs)

                # Reload data from disk.
                target = GDALRaster(target.name)
                self.assertEqual(target.srs.srid, 3086)
                self.assertEqual(target.width, 7)
                self.assertEqual(target.height, 7)
                self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype())
                self.assertAlmostEqual(target.origin[0], 9124842.791079799, 3)
                self.assertAlmostEqual(target.origin[1], 1589911.6476407414, 3)
                self.assertAlmostEqual(target.scale[0], 223824.82664250192, 3)
                self.assertAlmostEqual(target.scale[1], -223824.82664250192, 3)
                self.assertEqual(target.skew, [0, 0])

                result = target.bands[0].data()
                if numpy:
                    result = result.flatten().tolist()
                # The reprojection of a raster that spans over a large area
                # skews the data matrix and might introduce nodata values.
                self.assertEqual(
                    result,
                    [
                        ndv, ndv, ndv, ndv, 4, ndv, ndv,
                        ndv, ndv, 2, 3, 9, ndv, ndv,
                        ndv, 1, 2, 8, 13, 19, ndv,
                        0, 6, 6, 12, 18, 18, 24,
                        ndv, 10, 11, 16, 22, 23, ndv,
                        ndv, ndv, 15, 21, 22, ndv, ndv,
                        ndv, ndv, 20, ndv, ndv, ndv, ndv,
                    ],
                )

    def test_raster_transform_clone(self):
        with mock.patch.object(GDALRaster, 'clone') as mocked_clone:
            # Create in file based raster.
            rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
            source = GDALRaster({
                'datatype': 1,
                'driver': 'tif',
                'name': rstfile.name,
                'width': 5,
                'height': 5,
                'nr_of_bands': 1,
                'srid': 4326,
                'origin': (-5, 5),
                'scale': (2, -2),
                'skew': (0, 0),
                'bands': [{
                    'data': range(25),
                    'nodata_value': 99,
                }],
            })
            # transform() returns a clone because it is the same SRID and
            # driver.
            source.transform(4326)
            self.assertEqual(mocked_clone.call_count, 1)

    def test_raster_transform_clone_name(self):
        # Create in file based raster.
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
        source = GDALRaster({
            'datatype': 1,
            'driver': 'tif',
            'name': rstfile.name,
            'width': 5,
            'height': 5,
            'nr_of_bands': 1,
            'srid': 4326,
            'origin': (-5, 5),
            'scale': (2, -2),
            'skew': (0, 0),
            'bands': [{
                'data': range(25),
                'nodata_value': 99,
            }],
        })
        clone_name = rstfile.name + '_respect_name.GTiff'
        target = source.transform(4326, name=clone_name)
        self.assertEqual(target.name, clone_name)
Exemple #16
0
    def test_dwithin_gis_lookup_output_with_rasters(self):
        """
        Check the logical functionality of the dwithin lookup for different
        input parameters.
        """
        # Create test raster and geom.
        rast = GDALRaster(json.loads(JSON_RASTER))
        stx_pnt = GEOSGeometry("POINT (-95.370401017314293 29.704867409475465)", 4326)
        stx_pnt.transform(3086)

        # Filter raster with different lookup raster formats.
        qs = RasterModel.objects.filter(rastprojected__dwithin=(rast, D(km=1)))
        self.assertEqual(qs.count(), 1)

        qs = RasterModel.objects.filter(
            rastprojected__dwithin=(json.loads(JSON_RASTER), D(km=1))
        )
        self.assertEqual(qs.count(), 1)

        qs = RasterModel.objects.filter(rastprojected__dwithin=(JSON_RASTER, D(km=1)))
        self.assertEqual(qs.count(), 1)

        # Filter in an unprojected coordinate system.
        qs = RasterModel.objects.filter(rast__dwithin=(rast, 40))
        self.assertEqual(qs.count(), 1)

        # Filter with band index transform.
        qs = RasterModel.objects.filter(rast__1__dwithin=(rast, 1, 40))
        self.assertEqual(qs.count(), 1)
        qs = RasterModel.objects.filter(rast__1__dwithin=(rast, 40))
        self.assertEqual(qs.count(), 1)
        qs = RasterModel.objects.filter(rast__dwithin=(rast, 1, 40))
        self.assertEqual(qs.count(), 1)

        # Filter raster by geom.
        qs = RasterModel.objects.filter(rast__dwithin=(stx_pnt, 500))
        self.assertEqual(qs.count(), 1)

        qs = RasterModel.objects.filter(rastprojected__dwithin=(stx_pnt, D(km=10000)))
        self.assertEqual(qs.count(), 1)

        qs = RasterModel.objects.filter(rast__dwithin=(stx_pnt, 5))
        self.assertEqual(qs.count(), 0)

        qs = RasterModel.objects.filter(rastprojected__dwithin=(stx_pnt, D(km=100)))
        self.assertEqual(qs.count(), 0)

        # Filter geom by raster.
        qs = RasterModel.objects.filter(geom__dwithin=(rast, 500))
        self.assertEqual(qs.count(), 1)

        # Filter through related model.
        qs = RasterRelatedModel.objects.filter(rastermodel__rast__dwithin=(rast, 40))
        self.assertEqual(qs.count(), 1)

        # Filter through related model with band index transform
        qs = RasterRelatedModel.objects.filter(rastermodel__rast__1__dwithin=(rast, 40))
        self.assertEqual(qs.count(), 1)

        # Filter through conditional statements.
        qs = RasterModel.objects.filter(
            Q(rast__dwithin=(rast, 40))
            & Q(rastprojected__dwithin=(stx_pnt, D(km=10000)))
        )
        self.assertEqual(qs.count(), 1)

        # Filter through different lookup.
        qs = RasterModel.objects.filter(rastprojected__bbcontains=rast)
        self.assertEqual(qs.count(), 1)
Exemple #17
0
 def test_lookup_input_band_not_allowed(self):
     rast = GDALRaster(json.loads(JSON_RASTER))
     qs = RasterModel.objects.filter(rast__bbcontains=(rast, 1))
     msg = "Band indices are not allowed for this operator, it works on bbox only."
     with self.assertRaisesMessage(ValueError, msg):
         qs.count()
Exemple #18
0
    def test_raster_transform(self):
        # Prepare tempfile and nodata value
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
        ndv = 99

        # Create in file based raster
        source = GDALRaster({
            'datatype':
            1,
            'driver':
            'tif',
            'name':
            rstfile.name,
            'width':
            5,
            'height':
            5,
            'nr_of_bands':
            1,
            'srid':
            4326,
            'origin': (-5, 5),
            'scale': (2, -2),
            'skew': (0, 0),
            'bands': [{
                'data': range(25),
                'nodata_value': ndv,
            }],
        })

        # Transform raster into srid 4326.
        target = source.transform(3086)

        # Reload data from disk
        target = GDALRaster(target.name)

        self.assertEqual(target.srs.srid, 3086)
        self.assertEqual(target.width, 7)
        self.assertEqual(target.height, 7)
        self.assertEqual(target.bands[0].datatype(),
                         source.bands[0].datatype())
        self.assertAlmostEqual(target.origin[0], 9124842.791079799, 3)
        self.assertAlmostEqual(target.origin[1], 1589911.6476407414, 3)
        self.assertAlmostEqual(target.scale[0], 223824.82664250192, 3)
        self.assertAlmostEqual(target.scale[1], -223824.82664250192, 3)
        self.assertEqual(target.skew, [0, 0])

        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # The reprojection of a raster that spans over a large area
        # skews the data matrix and might introduce nodata values.
        self.assertEqual(result, [
            ndv,
            ndv,
            ndv,
            ndv,
            4,
            ndv,
            ndv,
            ndv,
            ndv,
            2,
            3,
            9,
            ndv,
            ndv,
            ndv,
            1,
            2,
            8,
            13,
            19,
            ndv,
            0,
            6,
            6,
            12,
            18,
            18,
            24,
            ndv,
            10,
            11,
            16,
            22,
            23,
            ndv,
            ndv,
            ndv,
            15,
            21,
            22,
            ndv,
            ndv,
            ndv,
            ndv,
            20,
            ndv,
            ndv,
            ndv,
            ndv,
        ])
Exemple #19
0
    def test_band_data_setters(self):
        # Create in-memory raster and get band
        rsmem = GDALRaster({
            'datatype': 1,
            'driver': 'MEM',
            'name': 'mem_rst',
            'width': 10,
            'height': 10,
            'nr_of_bands': 1,
            'srid': 4326,
        })
        bandmem = rsmem.bands[0]

        # Set nodata value
        bandmem.nodata_value = 99
        self.assertEqual(bandmem.nodata_value, 99)

        # Set data for entire dataset
        bandmem.data(range(100))
        if numpy:
            numpy.testing.assert_equal(bandmem.data(), numpy.arange(100).reshape(10, 10))
        else:
            self.assertEqual(bandmem.data(), list(range(100)))

        # Prepare data for setting values in subsequent tests
        block = list(range(100, 104))
        packed_block = struct.pack('<' + 'B B B B', *block)

        # Set data from list
        bandmem.data(block, (1, 1), (2, 2))
        result = bandmem.data(offset=(1, 1), size=(2, 2))
        if numpy:
            numpy.testing.assert_equal(result, numpy.array(block).reshape(2, 2))
        else:
            self.assertEqual(result, block)

        # Set data from packed block
        bandmem.data(packed_block, (1, 1), (2, 2))
        result = bandmem.data(offset=(1, 1), size=(2, 2))
        if numpy:
            numpy.testing.assert_equal(result, numpy.array(block).reshape(2, 2))
        else:
            self.assertEqual(result, block)

        # Set data from bytes
        bandmem.data(bytes(packed_block), (1, 1), (2, 2))
        result = bandmem.data(offset=(1, 1), size=(2, 2))
        if numpy:
            numpy.testing.assert_equal(result, numpy.array(block).reshape(2, 2))
        else:
            self.assertEqual(result, block)

        # Set data from bytearray
        bandmem.data(bytearray(packed_block), (1, 1), (2, 2))
        result = bandmem.data(offset=(1, 1), size=(2, 2))
        if numpy:
            numpy.testing.assert_equal(result, numpy.array(block).reshape(2, 2))
        else:
            self.assertEqual(result, block)

        # Set data from memoryview
        bandmem.data(memoryview(packed_block), (1, 1), (2, 2))
        result = bandmem.data(offset=(1, 1), size=(2, 2))
        if numpy:
            numpy.testing.assert_equal(result, numpy.array(block).reshape(2, 2))
        else:
            self.assertEqual(result, block)

        # Set data from numpy array
        if numpy:
            bandmem.data(numpy.array(block, dtype='int8').reshape(2, 2), (1, 1), (2, 2))
            numpy.testing.assert_equal(
                bandmem.data(offset=(1, 1), size=(2, 2)),
                numpy.array(block).reshape(2, 2)
            )

        # Test json input data
        rsmemjson = GDALRaster(JSON_RASTER)
        bandmemjson = rsmemjson.bands[0]
        if numpy:
            numpy.testing.assert_equal(
                bandmemjson.data(),
                numpy.array(range(25)).reshape(5, 5)
            )
        else:
            self.assertEqual(bandmemjson.data(), list(range(25)))
Exemple #20
0
 def test_gdalraster_input_as_path(self):
     rs_path = Path(
         __file__).parent.parent / "data" / "rasters" / "raster.tif"
     rs = GDALRaster(rs_path)
     self.assertEqual(str(rs_path), rs.name)
Exemple #21
0
    def evaluate_raster_algebra(self, data, formula, check_aligned=False):
        """
        Evaluate a raster algebra expression on a set of rasters. All input
        rasters need to be strictly aligned (same size, geotransform and srid).

        The input raster list will be zipped into a dictionary using the input
        names. The resulting dictionary will be used as input data for formula
        evaluation. If the check_aligned flag is set, the input rasters are
        compared to make sure they are aligned.
        """
        # Check that all input rasters are aligned
        if check_aligned:
            self.check_aligned(list(data.values()))

        # Construct list of numpy arrays holding raster pixel data, converting
        # data to default number type. This is necessary because formula
        # evaluation can lead to unexpected results. This could be
        # differentiated in the future based on data types and formulas.
        data_arrays = {}
        for key, rast in data.items():

            keysplit = key.split(const.BAND_INDEX_SEPARATOR)

            variable = keysplit[0]

            if len(keysplit) > 1:
                band_index = int(keysplit[1])
            else:
                band_index = 0

            if rast.bands[band_index].nodata_value is None:
                data_arrays[variable] = rast.bands[band_index].data().ravel(
                ).astype(const.ALGEBRA_PIXEL_TYPE_NUMPY)
            else:
                data_arrays[variable] = numpy.ma.masked_values(
                    rast.bands[band_index].data().ravel().astype(
                        const.ALGEBRA_PIXEL_TYPE_NUMPY),
                    rast.bands[band_index].nodata_value,
                )

        # Evaluate formula on raster data
        result = self.evaluate(data_arrays, formula)

        # Reference first original raster for constructing result
        orig = list(data.values())[0]

        # Get nodata value from mask or from original band data
        if numpy.ma.is_masked(result):
            # Get mask fill value
            nodata = float(result.fill_value)
            # Overwrite result with mask values filled in
            result = result.filled()
        else:
            nodata = orig.bands[0].nodata_value

        # Return GDALRaster holding results
        return GDALRaster({
            'datatype':
            const.ALGEBRA_PIXEL_TYPE_GDAL,
            'driver':
            'MEM',
            'width':
            orig.width,
            'height':
            orig.height,
            'nr_of_bands':
            1,
            'srid':
            orig.srs.srid,
            'origin':
            orig.origin,
            'scale':
            orig.scale,
            'skew':
            orig.skew,
            'bands': [{
                'nodata_value': nodata,
                'data': result,
            }],
        })
Exemple #22
0
class GDALRasterTests(SimpleTestCase):
    """
    Test a GDALRaster instance created from a file (GeoTiff).
    """
    def setUp(self):
        self.rs_path = os.path.join(os.path.dirname(__file__),
                                    "../data/rasters/raster.tif")
        self.rs = GDALRaster(self.rs_path)

    def test_gdalraster_input_as_path(self):
        rs_path = Path(
            __file__).parent.parent / "data" / "rasters" / "raster.tif"
        rs = GDALRaster(rs_path)
        self.assertEqual(str(rs_path), rs.name)

    def test_rs_name_repr(self):
        self.assertEqual(self.rs_path, self.rs.name)
        self.assertRegex(repr(self.rs), r"<Raster object at 0x\w+>")

    def test_rs_driver(self):
        self.assertEqual(self.rs.driver.name, "GTiff")

    def test_rs_size(self):
        self.assertEqual(self.rs.width, 163)
        self.assertEqual(self.rs.height, 174)

    def test_rs_srs(self):
        self.assertEqual(self.rs.srs.srid, 3086)
        self.assertEqual(self.rs.srs.units, (1.0, "metre"))

    def test_rs_srid(self):
        rast = GDALRaster({
            "width": 16,
            "height": 16,
            "srid": 4326,
        })
        self.assertEqual(rast.srid, 4326)
        rast.srid = 3086
        self.assertEqual(rast.srid, 3086)

    def test_geotransform_and_friends(self):
        # Assert correct values for file based raster
        self.assertEqual(
            self.rs.geotransform,
            [511700.4680706557, 100.0, 0.0, 435103.3771231986, 0.0, -100.0],
        )
        self.assertEqual(self.rs.origin,
                         [511700.4680706557, 435103.3771231986])
        self.assertEqual(self.rs.origin.x, 511700.4680706557)
        self.assertEqual(self.rs.origin.y, 435103.3771231986)
        self.assertEqual(self.rs.scale, [100.0, -100.0])
        self.assertEqual(self.rs.scale.x, 100.0)
        self.assertEqual(self.rs.scale.y, -100.0)
        self.assertEqual(self.rs.skew, [0, 0])
        self.assertEqual(self.rs.skew.x, 0)
        self.assertEqual(self.rs.skew.y, 0)
        # Create in-memory rasters and change gtvalues
        rsmem = GDALRaster(JSON_RASTER)
        # geotransform accepts both floats and ints
        rsmem.geotransform = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
        self.assertEqual(rsmem.geotransform, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0])
        rsmem.geotransform = range(6)
        self.assertEqual(rsmem.geotransform, [float(x) for x in range(6)])
        self.assertEqual(rsmem.origin, [0, 3])
        self.assertEqual(rsmem.origin.x, 0)
        self.assertEqual(rsmem.origin.y, 3)
        self.assertEqual(rsmem.scale, [1, 5])
        self.assertEqual(rsmem.scale.x, 1)
        self.assertEqual(rsmem.scale.y, 5)
        self.assertEqual(rsmem.skew, [2, 4])
        self.assertEqual(rsmem.skew.x, 2)
        self.assertEqual(rsmem.skew.y, 4)
        self.assertEqual(rsmem.width, 5)
        self.assertEqual(rsmem.height, 5)

    def test_geotransform_bad_inputs(self):
        rsmem = GDALRaster(JSON_RASTER)
        error_geotransforms = [
            [1, 2],
            [1, 2, 3, 4, 5, "foo"],
            [1, 2, 3, 4, 5, 6, "foo"],
        ]
        msg = "Geotransform must consist of 6 numeric values."
        for geotransform in error_geotransforms:
            with self.subTest(i=geotransform), self.assertRaisesMessage(
                    ValueError, msg):
                rsmem.geotransform = geotransform

    def test_rs_extent(self):
        self.assertEqual(
            self.rs.extent,
            (
                511700.4680706557,
                417703.3771231986,
                528000.4680706557,
                435103.3771231986,
            ),
        )

    def test_rs_bands(self):
        self.assertEqual(len(self.rs.bands), 1)
        self.assertIsInstance(self.rs.bands[0], GDALBand)

    def test_memory_based_raster_creation(self):
        # Create uint8 raster with full pixel data range (0-255)
        rast = GDALRaster({
            "datatype":
            1,
            "width":
            16,
            "height":
            16,
            "srid":
            4326,
            "bands": [{
                "data": range(256),
                "nodata_value": 255,
            }],
        })

        # Get array from raster
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # Assert data is same as original input
        self.assertEqual(result, list(range(256)))

    def test_file_based_raster_creation(self):
        # Prepare tempfile
        rstfile = tempfile.NamedTemporaryFile(suffix=".tif")

        # Create file-based raster from scratch
        GDALRaster({
            "datatype":
            self.rs.bands[0].datatype(),
            "driver":
            "tif",
            "name":
            rstfile.name,
            "width":
            163,
            "height":
            174,
            "nr_of_bands":
            1,
            "srid":
            self.rs.srs.wkt,
            "origin": (self.rs.origin.x, self.rs.origin.y),
            "scale": (self.rs.scale.x, self.rs.scale.y),
            "skew": (self.rs.skew.x, self.rs.skew.y),
            "bands": [{
                "data": self.rs.bands[0].data(),
                "nodata_value": self.rs.bands[0].nodata_value,
            }],
        })

        # Reload newly created raster from file
        restored_raster = GDALRaster(rstfile.name)
        # Presence of TOWGS84 depend on GDAL/Proj versions.
        self.assertEqual(
            restored_raster.srs.wkt.replace("TOWGS84[0,0,0,0,0,0,0],", ""),
            self.rs.srs.wkt.replace("TOWGS84[0,0,0,0,0,0,0],", ""),
        )
        self.assertEqual(restored_raster.geotransform, self.rs.geotransform)
        if numpy:
            numpy.testing.assert_equal(restored_raster.bands[0].data(),
                                       self.rs.bands[0].data())
        else:
            self.assertEqual(restored_raster.bands[0].data(),
                             self.rs.bands[0].data())

    def test_nonexistent_file(self):
        msg = 'Unable to read raster source input "nonexistent.tif".'
        with self.assertRaisesMessage(GDALException, msg):
            GDALRaster("nonexistent.tif")

    def test_vsi_raster_creation(self):
        # Open a raster as a file object.
        with open(self.rs_path, "rb") as dat:
            # Instantiate a raster from the file binary buffer.
            vsimem = GDALRaster(dat.read())
        # The data of the in-memory file is equal to the source file.
        result = vsimem.bands[0].data()
        target = self.rs.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
            target = target.flatten().tolist()
        self.assertEqual(result, target)

    def test_vsi_raster_deletion(self):
        path = "/vsimem/raster.tif"
        # Create a vsi-based raster from scratch.
        vsimem = GDALRaster({
            "name": path,
            "driver": "tif",
            "width": 4,
            "height": 4,
            "srid": 4326,
            "bands": [{
                "data": range(16),
            }],
        })
        # The virtual file exists.
        rst = GDALRaster(path)
        self.assertEqual(rst.width, 4)
        # Delete GDALRaster.
        del vsimem
        del rst
        # The virtual file has been removed.
        msg = 'Could not open the datasource at "/vsimem/raster.tif"'
        with self.assertRaisesMessage(GDALException, msg):
            GDALRaster(path)

    def test_vsi_invalid_buffer_error(self):
        msg = "Failed creating VSI raster from the input buffer."
        with self.assertRaisesMessage(GDALException, msg):
            GDALRaster(b"not-a-raster-buffer")

    def test_vsi_buffer_property(self):
        # Create a vsi-based raster from scratch.
        rast = GDALRaster({
            "name": "/vsimem/raster.tif",
            "driver": "tif",
            "width": 4,
            "height": 4,
            "srid": 4326,
            "bands": [{
                "data": range(16),
            }],
        })
        # Do a round trip from raster to buffer to raster.
        result = GDALRaster(rast.vsi_buffer).bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to nodata value except on input block of ones.
        self.assertEqual(result, list(range(16)))
        # The vsi buffer is None for rasters that are not vsi based.
        self.assertIsNone(self.rs.vsi_buffer)

    def test_vsi_vsizip_filesystem(self):
        rst_zipfile = tempfile.NamedTemporaryFile(suffix=".zip")
        with zipfile.ZipFile(rst_zipfile, mode="w") as zf:
            zf.write(self.rs_path, "raster.tif")
        rst_path = "/vsizip/" + os.path.join(rst_zipfile.name, "raster.tif")
        rst = GDALRaster(rst_path)
        self.assertEqual(rst.driver.name, self.rs.driver.name)
        self.assertEqual(rst.name, rst_path)
        self.assertIs(rst.is_vsi_based, True)
        self.assertIsNone(rst.vsi_buffer)

    def test_offset_size_and_shape_on_raster_creation(self):
        rast = GDALRaster({
            "datatype":
            1,
            "width":
            4,
            "height":
            4,
            "srid":
            4326,
            "bands": [{
                "data": (1, ),
                "offset": (1, 1),
                "size": (2, 2),
                "shape": (1, 1),
                "nodata_value": 2,
            }],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to nodata value except on input block of ones.
        self.assertEqual(result,
                         [2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2])

    def test_set_nodata_value_on_raster_creation(self):
        # Create raster filled with nodata values.
        rast = GDALRaster({
            "datatype": 1,
            "width": 2,
            "height": 2,
            "srid": 4326,
            "bands": [{
                "nodata_value": 23
            }],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # All band data is equal to nodata value.
        self.assertEqual(result, [23] * 4)

    def test_set_nodata_none_on_raster_creation(self):
        # Create raster without data and without nodata value.
        rast = GDALRaster({
            "datatype": 1,
            "width": 2,
            "height": 2,
            "srid": 4326,
            "bands": [{
                "nodata_value": None
            }],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to zero because no nodata value has been specified.
        self.assertEqual(result, [0] * 4)

    def test_raster_metadata_property(self):
        data = self.rs.metadata
        self.assertEqual(data["DEFAULT"], {"AREA_OR_POINT": "Area"})
        self.assertEqual(data["IMAGE_STRUCTURE"], {"INTERLEAVE": "BAND"})

        # Create file-based raster from scratch
        source = GDALRaster({
            "datatype": 1,
            "width": 2,
            "height": 2,
            "srid": 4326,
            "bands": [{
                "data": range(4),
                "nodata_value": 99
            }],
        })
        # Set metadata on raster and on a band.
        metadata = {
            "DEFAULT": {
                "OWNER": "Django",
                "VERSION": "1.0",
                "AREA_OR_POINT": "Point"
            },
        }
        source.metadata = metadata
        source.bands[0].metadata = metadata
        self.assertEqual(source.metadata["DEFAULT"], metadata["DEFAULT"])
        self.assertEqual(source.bands[0].metadata["DEFAULT"],
                         metadata["DEFAULT"])
        # Update metadata on raster.
        metadata = {
            "DEFAULT": {
                "VERSION": "2.0"
            },
        }
        source.metadata = metadata
        self.assertEqual(source.metadata["DEFAULT"]["VERSION"], "2.0")
        # Remove metadata on raster.
        metadata = {
            "DEFAULT": {
                "OWNER": None
            },
        }
        source.metadata = metadata
        self.assertNotIn("OWNER", source.metadata["DEFAULT"])

    def test_raster_info_accessor(self):
        infos = self.rs.info
        # Data
        info_lines = [
            line.strip() for line in infos.split("\n") if line.strip() != ""
        ]
        for line in [
                "Driver: GTiff/GeoTIFF",
                "Files: {}".format(self.rs_path),
                "Size is 163, 174",
                "Origin = (511700.468070655711927,435103.377123198588379)",
                "Pixel Size = (100.000000000000000,-100.000000000000000)",
                "Metadata:",
                "AREA_OR_POINT=Area",
                "Image Structure Metadata:",
                "INTERLEAVE=BAND",
                "Band 1 Block=163x50 Type=Byte, ColorInterp=Gray",
                "NoData Value=15",
        ]:
            self.assertIn(line, info_lines)
        for line in [
                r"Upper Left  \(  511700.468,  435103.377\) "
                r'\( 82d51\'46.1\d"W, 27d55\' 1.5\d"N\)',
                r"Lower Left  \(  511700.468,  417703.377\) "
                r'\( 82d51\'52.0\d"W, 27d45\'37.5\d"N\)',
                r"Upper Right \(  528000.468,  435103.377\) "
                r'\( 82d41\'48.8\d"W, 27d54\'56.3\d"N\)',
                r"Lower Right \(  528000.468,  417703.377\) "
                r'\( 82d41\'55.5\d"W, 27d45\'32.2\d"N\)',
                r"Center      \(  519850.468,  426403.377\) "
                r'\( 82d46\'50.6\d"W, 27d50\'16.9\d"N\)',
        ]:
            self.assertRegex(infos, line)
        # CRS (skip the name because string depends on the GDAL/Proj versions).
        self.assertIn("NAD83 / Florida GDL Albers", infos)

    def test_compressed_file_based_raster_creation(self):
        rstfile = tempfile.NamedTemporaryFile(suffix=".tif")
        # Make a compressed copy of an existing raster.
        compressed = self.rs.warp({
            "papsz_options": {
                "compress": "packbits"
            },
            "name": rstfile.name
        })
        # Check physically if compression worked.
        self.assertLess(os.path.getsize(compressed.name),
                        os.path.getsize(self.rs.name))
        # Create file-based raster with options from scratch.
        compressed = GDALRaster({
            "datatype":
            1,
            "driver":
            "tif",
            "name":
            rstfile.name,
            "width":
            40,
            "height":
            40,
            "srid":
            3086,
            "origin": (500000, 400000),
            "scale": (100, -100),
            "skew": (0, 0),
            "bands": [{
                "data": range(40 ^ 2),
                "nodata_value": 255,
            }],
            "papsz_options": {
                "compress": "packbits",
                "pixeltype": "signedbyte",
                "blockxsize": 23,
                "blockysize": 23,
            },
        })
        # Check if options used on creation are stored in metadata.
        # Reopening the raster ensures that all metadata has been written
        # to the file.
        compressed = GDALRaster(compressed.name)
        self.assertEqual(
            compressed.metadata["IMAGE_STRUCTURE"]["COMPRESSION"],
            "PACKBITS",
        )
        self.assertEqual(
            compressed.bands[0].metadata["IMAGE_STRUCTURE"]["PIXELTYPE"],
            "SIGNEDBYTE")
        self.assertIn("Block=40x23", compressed.info)

    def test_raster_warp(self):
        # Create in memory raster
        source = GDALRaster({
            "datatype":
            1,
            "driver":
            "MEM",
            "name":
            "sourceraster",
            "width":
            4,
            "height":
            4,
            "nr_of_bands":
            1,
            "srid":
            3086,
            "origin": (500000, 400000),
            "scale": (100, -100),
            "skew": (0, 0),
            "bands": [{
                "data": range(16),
                "nodata_value": 255,
            }],
        })

        # Test altering the scale, width, and height of a raster
        data = {
            "scale": [200, -200],
            "width": 2,
            "height": 2,
        }
        target = source.warp(data)
        self.assertEqual(target.width, data["width"])
        self.assertEqual(target.height, data["height"])
        self.assertEqual(target.scale, data["scale"])
        self.assertEqual(target.bands[0].datatype(),
                         source.bands[0].datatype())
        self.assertEqual(target.name, "sourceraster_copy.MEM")
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(result, [5, 7, 13, 15])

        # Test altering the name and datatype (to float)
        data = {
            "name": "/path/to/targetraster.tif",
            "datatype": 6,
        }
        target = source.warp(data)
        self.assertEqual(target.bands[0].datatype(), 6)
        self.assertEqual(target.name, "/path/to/targetraster.tif")
        self.assertEqual(target.driver.name, "MEM")
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(
            result,
            [
                0.0,
                1.0,
                2.0,
                3.0,
                4.0,
                5.0,
                6.0,
                7.0,
                8.0,
                9.0,
                10.0,
                11.0,
                12.0,
                13.0,
                14.0,
                15.0,
            ],
        )

    def test_raster_warp_nodata_zone(self):
        # Create in memory raster.
        source = GDALRaster({
            "datatype":
            1,
            "driver":
            "MEM",
            "width":
            4,
            "height":
            4,
            "srid":
            3086,
            "origin": (500000, 400000),
            "scale": (100, -100),
            "skew": (0, 0),
            "bands": [{
                "data": range(16),
                "nodata_value": 23,
            }],
        })
        # Warp raster onto a location that does not cover any pixels of the original.
        result = source.warp({"origin": (200000, 200000)}).bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # The result is an empty raster filled with the correct nodata value.
        self.assertEqual(result, [23] * 16)

    def test_raster_clone(self):
        rstfile = tempfile.NamedTemporaryFile(suffix=".tif")
        tests = [
            ("MEM", "", 23),  # In memory raster.
            ("tif", rstfile.name, 99),  # In file based raster.
        ]
        for driver, name, nodata_value in tests:
            with self.subTest(driver=driver):
                source = GDALRaster({
                    "datatype":
                    1,
                    "driver":
                    driver,
                    "name":
                    name,
                    "width":
                    4,
                    "height":
                    4,
                    "srid":
                    3086,
                    "origin": (500000, 400000),
                    "scale": (100, -100),
                    "skew": (0, 0),
                    "bands": [{
                        "data": range(16),
                        "nodata_value": nodata_value,
                    }],
                })
                clone = source.clone()
                self.assertNotEqual(clone.name, source.name)
                self.assertEqual(clone._write, source._write)
                self.assertEqual(clone.srs.srid, source.srs.srid)
                self.assertEqual(clone.width, source.width)
                self.assertEqual(clone.height, source.height)
                self.assertEqual(clone.origin, source.origin)
                self.assertEqual(clone.scale, source.scale)
                self.assertEqual(clone.skew, source.skew)
                self.assertIsNot(clone, source)

    def test_raster_transform(self):
        tests = [
            3086,
            "3086",
            SpatialReference(3086),
        ]
        for srs in tests:
            with self.subTest(srs=srs):
                # Prepare tempfile and nodata value.
                rstfile = tempfile.NamedTemporaryFile(suffix=".tif")
                ndv = 99
                # Create in file based raster.
                source = GDALRaster({
                    "datatype":
                    1,
                    "driver":
                    "tif",
                    "name":
                    rstfile.name,
                    "width":
                    5,
                    "height":
                    5,
                    "nr_of_bands":
                    1,
                    "srid":
                    4326,
                    "origin": (-5, 5),
                    "scale": (2, -2),
                    "skew": (0, 0),
                    "bands": [{
                        "data": range(25),
                        "nodata_value": ndv,
                    }],
                })

                target = source.transform(srs)

                # Reload data from disk.
                target = GDALRaster(target.name)
                self.assertEqual(target.srs.srid, 3086)
                self.assertEqual(target.width, 7)
                self.assertEqual(target.height, 7)
                self.assertEqual(target.bands[0].datatype(),
                                 source.bands[0].datatype())
                self.assertAlmostEqual(target.origin[0], 9124842.791079799, 3)
                self.assertAlmostEqual(target.origin[1], 1589911.6476407414, 3)
                self.assertAlmostEqual(target.scale[0], 223824.82664250192, 3)
                self.assertAlmostEqual(target.scale[1], -223824.82664250192, 3)
                self.assertEqual(target.skew, [0, 0])

                result = target.bands[0].data()
                if numpy:
                    result = result.flatten().tolist()
                # The reprojection of a raster that spans over a large area
                # skews the data matrix and might introduce nodata values.
                self.assertEqual(
                    result,
                    [
                        ndv,
                        ndv,
                        ndv,
                        ndv,
                        4,
                        ndv,
                        ndv,
                        ndv,
                        ndv,
                        2,
                        3,
                        9,
                        ndv,
                        ndv,
                        ndv,
                        1,
                        2,
                        8,
                        13,
                        19,
                        ndv,
                        0,
                        6,
                        6,
                        12,
                        18,
                        18,
                        24,
                        ndv,
                        10,
                        11,
                        16,
                        22,
                        23,
                        ndv,
                        ndv,
                        ndv,
                        15,
                        21,
                        22,
                        ndv,
                        ndv,
                        ndv,
                        ndv,
                        20,
                        ndv,
                        ndv,
                        ndv,
                        ndv,
                    ],
                )

    def test_raster_transform_clone(self):
        with mock.patch.object(GDALRaster, "clone") as mocked_clone:
            # Create in file based raster.
            rstfile = tempfile.NamedTemporaryFile(suffix=".tif")
            source = GDALRaster({
                "datatype":
                1,
                "driver":
                "tif",
                "name":
                rstfile.name,
                "width":
                5,
                "height":
                5,
                "nr_of_bands":
                1,
                "srid":
                4326,
                "origin": (-5, 5),
                "scale": (2, -2),
                "skew": (0, 0),
                "bands": [{
                    "data": range(25),
                    "nodata_value": 99,
                }],
            })
            # transform() returns a clone because it is the same SRID and
            # driver.
            source.transform(4326)
            self.assertEqual(mocked_clone.call_count, 1)

    def test_raster_transform_clone_name(self):
        # Create in file based raster.
        rstfile = tempfile.NamedTemporaryFile(suffix=".tif")
        source = GDALRaster({
            "datatype":
            1,
            "driver":
            "tif",
            "name":
            rstfile.name,
            "width":
            5,
            "height":
            5,
            "nr_of_bands":
            1,
            "srid":
            4326,
            "origin": (-5, 5),
            "scale": (2, -2),
            "skew": (0, 0),
            "bands": [{
                "data": range(25),
                "nodata_value": 99,
            }],
        })
        clone_name = rstfile.name + "_respect_name.GTiff"
        target = source.transform(4326, name=clone_name)
        self.assertEqual(target.name, clone_name)
Exemple #23
0
 def setUp(self):
     self.rs_path = os.path.join(os.path.dirname(__file__),
                                 "../data/rasters/raster.tif")
     self.rs = GDALRaster(self.rs_path)
Exemple #24
0
    def process_quadrant(self, indexrange, zoom):
        """
        Create raster tiles for a quadrant of tiles defined by a x-y-z index
        range and a zoom level.
        """
        # TODO Use a standalone celery task for this method in order to
        # gain speedup from parallelism.
        self._quadrant_count += 1
        self.log(
            'Starting tile creation for quadrant {0} at zoom level {1}'.format(self._quadrant_count, zoom),
            status=self.rasterlayer.parsestatus.CREATING_TILES
        )

        # Compute scale of tiles for this zoomlevel
        tilescale = utils.tile_scale(zoom)

        # Compute quadrant bounds and create destination file
        bounds = utils.tile_bounds(indexrange[0], indexrange[1], zoom)
        dest_file = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.tif')

        # Snap dataset to the quadrant
        snapped_dataset = self.dataset.warp({
            'name': dest_file.name,
            'origin': [bounds[0], bounds[3]],
            'scale': [tilescale, -tilescale],
            'width': (indexrange[2] - indexrange[0] + 1) * self.tilesize,
            'height': (indexrange[3] - indexrange[1] + 1) * self.tilesize,
        })

        # Create all tiles in this quadrant in batches
        batch = []
        for tilex in range(indexrange[0], indexrange[2] + 1):
            for tiley in range(indexrange[1], indexrange[3] + 1):
                # Calculate raster tile origin
                bounds = utils.tile_bounds(tilex, tiley, zoom)

                # Construct band data arrays
                pixeloffset = (
                    (tilex - indexrange[0]) * self.tilesize,
                    (tiley - indexrange[1]) * self.tilesize
                )

                band_data = [
                    {
                        'data': band.data(offset=pixeloffset, size=(self.tilesize, self.tilesize)),
                        'nodata_value': band.nodata_value
                    } for band in snapped_dataset.bands
                ]

                # Ignore tile if its only nodata.
                if all([numpy.all(dat['data'] == dat['nodata_value']) for dat in band_data]):
                    continue

                # Add tile data to histogram
                if zoom == self.max_zoom:
                    self.push_histogram(band_data)

                # Warp source raster into this tile (in memory)
                dest = GDALRaster({
                    'width': self.tilesize,
                    'height': self.tilesize,
                    'origin': [bounds[0], bounds[3]],
                    'scale': [tilescale, -tilescale],
                    'srid': WEB_MERCATOR_SRID,
                    'datatype': snapped_dataset.bands[0].datatype(),
                    'bands': band_data,
                })

                # Store tile in batch array
                batch.append(
                    RasterTile(
                        rast=dest,
                        rasterlayer_id=self.rasterlayer.id,
                        tilex=tilex,
                        tiley=tiley,
                        tilez=zoom
                    )
                )

                # Commit batch to database and reset it
                if len(batch) == self.batch_step_size:
                    RasterTile.objects.bulk_create(batch)
                    batch = []

        # Commit remaining objects
        if len(batch):
            RasterTile.objects.bulk_create(batch)
Exemple #25
0
    def open_raster_file(self):
        """
        Get raster source file to extract tiles from.

        This makes a local copy of rasterfile, unzips the raster and reprojects
        it into web mercator if necessary. The reprojected raster is stored for
        reuse such that reprojection does only happen once.

        The local copy of the raster is needed if files are stored on remote
        storages.
        """
        reproj, created = RasterLayerReprojected.objects.get_or_create(rasterlayer=self.rasterlayer)
        # Check if the raster has already been reprojected
        has_reprojected = reproj.rasterfile.name not in (None, '')

        # Create workdir
        raster_workdir = getattr(settings, 'RASTER_WORKDIR', None)
        self.tmpdir = tempfile.mkdtemp(dir=raster_workdir)

        # Choose source for raster data, use the reprojected version if it exists.
        if self.rasterlayer.source_url and not has_reprojected:
            url = self.rasterlayer.source_url
            if url.lower().startswith('http') or url.startswith('file'):
                url_path = urlparse(self.rasterlayer.source_url).path
                filename = url_path.split('/')[-1]
                filepath = os.path.join(self.tmpdir, filename)
                urlretrieve(self.rasterlayer.source_url, filepath)
            elif url.startswith('s3'):
                # Get the bucket name and file key, assuming the following url
                # strucure: s3://BUCKET_NAME/BUCKET_KEY
                bucket_name = url.split('s3://')[1].split('/')[0]
                bucket_key = '/'.join(url.split('s3://')[1].split('/')[1:])
                # Assume the file name is the last piece of the key.
                filename = bucket_key.split('/')[-1]
                filepath = os.path.join(self.tmpdir, filename)
                # Get file from s3.
                s3 = boto3.resource('s3')
                bucket = s3.Bucket(bucket_name)
                bucket.download_file(bucket_key, filepath, ExtraArgs={'RequestPayer': 'requester'})
            else:
                raise RasterException('Only http(s) and s3 urls are supported.')
        else:
            if has_reprojected:
                rasterfile_source = reproj.rasterfile
            else:
                rasterfile_source = self.rasterlayer.rasterfile

            if not rasterfile_source.name:
                raise RasterException('No data source found. Provide a rasterfile or a source url.')

            # Copy raster file source to local folder
            filepath = os.path.join(self.tmpdir, os.path.basename(rasterfile_source.name))
            rasterfile = open(filepath, 'wb')
            for chunk in rasterfile_source.chunks():
                rasterfile.write(chunk)
            rasterfile.close()

        # If the raster file is compressed, decompress it, otherwise try to
        # open the source file directly.
        if os.path.splitext(filepath)[1].lower() == '.zip':
            # Open and extract zipfile
            zf = zipfile.ZipFile(filepath)
            zf.extractall(self.tmpdir)

            # Remove zipfile
            os.remove(filepath)

            # Get filelist from directory
            matches = []
            for root, dirnames, filenames in os.walk(self.tmpdir):
                for filename in fnmatch.filter(filenames, '*.*'):
                    matches.append(os.path.join(root, filename))

            # Open the first raster file found in the matched files.
            self.dataset = None
            for match in matches:
                try:
                    self.dataset = GDALRaster(match)
                    break
                except GDALException:
                    pass

            # Raise exception if no file could be opened by gdal.
            if not self.dataset:
                raise RasterException('Could not open rasterfile.')
        else:
            self.dataset = GDALRaster(filepath)

        # Override srid if provided
        if self.rasterlayer.srid:
            try:
                self.dataset = GDALRaster(self.dataset.name, write=True)
            except GDALException:
                raise RasterException(
                    'Could not override srid because the driver for this '
                    'type of raster does not support write mode.'
                )
            self.dataset.srs = self.rasterlayer.srid
Exemple #26
0
class RasterLayerParser(object):
    """
    Class to parse raster layers.
    """

    def __init__(self, rasterlayer):
        self.rasterlayer = rasterlayer
        self.rastername = os.path.basename(rasterlayer.rasterfile.name)

        # Set raster tilesize
        self.tilesize = int(getattr(settings, "RASTER_TILESIZE", WEB_MERCATOR_TILESIZE))
        self.zoomdown = getattr(settings, "RASTER_ZOOM_NEXT_HIGHER", True)

    def log(self, msg, reset=False, status=None, zoom=None):
        """
        Write a message to the parse log of the rasterlayer instance and update
        the parse status object.
        """
        if status is not None:
            self.rasterlayer.parsestatus.status = status

        if zoom is not None:
            self.rasterlayer.parsestatus.tile_level = zoom

        # Prepare datetime stamp for log
        now = "[{0}] ".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))

        # Write log, reset if requested
        if reset:
            self.rasterlayer.parsestatus.log = now + msg
        else:
            self.rasterlayer.parsestatus.log += "\n" + now + msg

        self.rasterlayer.save()
        self.rasterlayer.parsestatus.save()

    def get_raster_file(self):
        """
        Make local copy of rasterfile, which is needed if files are stored on
        remote storage, and unzip it if necessary.
        """
        self.log("Getting raster file from storage")

        raster_workdir = getattr(settings, "RASTER_WORKDIR", None)
        self.tmpdir = tempfile.mkdtemp(dir=raster_workdir)

        # Access rasterfile and store in a temp folder
        rasterfile = open(os.path.join(self.tmpdir, self.rastername), "wb")
        for chunk in self.rasterlayer.rasterfile.chunks():
            rasterfile.write(chunk)
        rasterfile.close()

        # If the raster file is compressed, decompress it
        fileName, fileExtension = os.path.splitext(self.rastername)

        if fileExtension == ".zip":

            # Open and extract zipfile
            zf = zipfile.ZipFile(os.path.join(self.tmpdir, self.rastername))
            zf.extractall(self.tmpdir)

            # Remove zipfile
            os.remove(os.path.join(self.tmpdir, self.rastername))

            # Get filelist from directory
            raster_list = glob.glob(os.path.join(self.tmpdir, "*.*"))

            # Check if only one file is found in zipfile
            if len(raster_list) > 1:
                self.log(
                    "WARNING: Found more than one file in zipfile "
                    "using only first file found. This might lead "
                    "to problems if its not a raster file."
                )

            # Return first one as raster file
            self.rastername = os.path.basename(raster_list[0])

    def open_raster_file(self):
        """
        Open the raster file as GDALRaster and set nodata-values.
        """
        self.log("Opening raster file as GDALRaster.")

        # Open raster file
        self.dataset = GDALRaster(os.path.join(self.tmpdir, self.rastername), write=True)

        # Make sure nodata value is set from input
        self.hist_values = []
        self.hist_bins = []
        for i, band in enumerate(self.dataset.bands):
            if self.rasterlayer.nodata is not None:
                band.nodata_value = float(self.rasterlayer.nodata)

            # Create band metatdata object
            bandmeta = RasterLayerBandMetadata.objects.create(
                rasterlayer=self.rasterlayer, band=i, nodata_value=band.nodata_value, min=band.min, max=band.max
            )

            # Prepare numpy hist values and bins
            self.hist_values.append(numpy.array(bandmeta.hist_values))
            self.hist_bins.append(numpy.array(bandmeta.hist_bins))

        # Store original metadata for this raster
        meta = self.rasterlayer.metadata

        meta.uperleftx = self.dataset.origin.x
        meta.uperlefty = self.dataset.origin.y
        meta.width = self.dataset.width
        meta.height = self.dataset.height
        meta.scalex = self.dataset.scale.x
        meta.scaley = self.dataset.scale.y
        meta.skewx = self.dataset.skew.x
        meta.skewy = self.dataset.skew.y
        meta.numbands = len(self.dataset.bands)
        meta.srs_wkt = self.dataset.srs.wkt
        meta.srid = self.dataset.srs.srid

        meta.save()

    def close_raster_file(self):
        """
        On Windows close and release the GDALRaster resources
        """
        try:
            if self.dataset:
                del self.dataset
                self.dataset = None
        except AttributeError:
            pass

    def create_tiles(self, zoom):
        """
        Create tiles for this raster at the given zoomlevel.

        This routine first snaps the raster to the grid of the zoomlevel,
        then creates  the tiles from the snapped raster.
        """
        # Compute the tile x-y-z index range for the rasterlayer for this zoomlevel
        bbox = self.rasterlayer.extent()
        indexrange = tiler.tile_index_range(bbox, zoom)

        # Compute scale of tiles for this zoomlevel
        tilescale = tiler.tile_scale(zoom)

        # Count the number of tiles that are required to cover the raster at this zoomlevel
        nr_of_tiles = (indexrange[2] - indexrange[0] + 1) * (indexrange[3] - indexrange[1] + 1)

        # Create destination raster file
        self.log("Snapping dataset to zoom level {0}".format(zoom))

        bounds = tiler.tile_bounds(indexrange[0], indexrange[1], zoom)
        sizex = (indexrange[2] - indexrange[0] + 1) * self.tilesize
        sizey = (indexrange[3] - indexrange[1] + 1) * self.tilesize
        dest_file = os.path.join(self.tmpdir, "djangowarpedraster" + str(zoom) + ".tif")

        snapped_dataset = self.dataset.warp(
            {
                "name": dest_file,
                "origin": [bounds[0], bounds[3]],
                "scale": [tilescale, -tilescale],
                "width": sizex,
                "height": sizey,
            }
        )

        self.log("Creating {0} tiles for zoom {1}.".format(nr_of_tiles, zoom))

        counter = 0
        for tilex in range(indexrange[0], indexrange[2] + 1):
            for tiley in range(indexrange[1], indexrange[3] + 1):
                # Log progress
                counter += 1
                if counter % 250 == 0:
                    self.log("{0} tiles created at zoom {1}".format(counter, zoom))

                # Calculate raster tile origin
                bounds = tiler.tile_bounds(tilex, tiley, zoom)

                # Construct band data arrays
                pixeloffset = ((tilex - indexrange[0]) * self.tilesize, (tiley - indexrange[1]) * self.tilesize)

                band_data = [
                    {
                        "data": band.data(offset=pixeloffset, size=(self.tilesize, self.tilesize)),
                        "nodata_value": band.nodata_value,
                    }
                    for band in snapped_dataset.bands
                ]

                # Add tile data to histogram
                if zoom == self.max_zoom:
                    self.push_histogram(band_data)

                # Warp source raster into this tile (in memory)
                dest = GDALRaster(
                    {
                        "width": self.tilesize,
                        "height": self.tilesize,
                        "origin": [bounds[0], bounds[3]],
                        "scale": [tilescale, -tilescale],
                        "srid": WEB_MERCATOR_SRID,
                        "datatype": snapped_dataset.bands[0].datatype(),
                        "bands": band_data,
                    }
                )

                # Store tile
                RasterTile.objects.create(rast=dest, rasterlayer=self.rasterlayer, tilex=tilex, tiley=tiley, tilez=zoom)

        # Store histogram data
        if zoom == self.max_zoom:
            bandmetas = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer)
            for bandmeta in bandmetas:
                bandmeta.hist_values = self.hist_values[bandmeta.band].tolist()
                bandmeta.save()

        # Remove snapped dataset
        self.log("Removing snapped dataset.", zoom=zoom)
        snapped_dataset = None
        os.remove(dest_file)

    def push_histogram(self, data):
        """
        Add data to band level histogram histogram.
        """
        # Loop through bands of this tile
        for i, dat in enumerate(data):
            # Create histogram for new data with the same bins
            new_hist = numpy.histogram(dat["data"], bins=self.hist_bins[i])
            # Add counts of this tile to band metadata histogram
            self.hist_values[i] += new_hist[0]

    def drop_empty_rasters(self):
        """
        Remove rasters that are only no-data from the current rasterlayer.
        """
        self.log("Dropping empty raster tiles.", status=self.rasterlayer.parsestatus.DROPPING_EMPTY_TILES)

        # Setup SQL command
        sql = ("DELETE FROM raster_rastertile " "WHERE ST_Count(rast)=0 " "AND rasterlayer_id={0}").format(
            self.rasterlayer.id
        )

        # Run SQL to drop empty tiles
        cursor = connection.cursor()
        cursor.execute(sql)

    def parse_raster_layer(self):
        """
        This function pushes the raster data from the Raster Layer into the
        RasterTile table.
        """
        try:
            # Clean previous parse log
            self.log("Started parsing raster file", reset=True, status=self.rasterlayer.parsestatus.DOWNLOADING_FILE)

            # Download, unzip and open raster file
            self.get_raster_file()
            self.open_raster_file()

            # Remove existing tiles for this layer before loading new ones
            self.rasterlayer.rastertile_set.all().delete()

            # Transform raster to global srid
            if self.dataset.srs.srid == WEB_MERCATOR_SRID:
                self.log("Dataset already in SRID {0}, skipping transform".format(WEB_MERCATOR_SRID))
            else:
                self.log(
                    "Transforming raster to SRID {0}".format(WEB_MERCATOR_SRID),
                    status=self.rasterlayer.parsestatus.REPROJECTING_RASTER,
                )
                self.dataset = self.dataset.transform(WEB_MERCATOR_SRID)

            # Compute max zoom at the web mercator projection
            self.max_zoom = tiler.closest_zoomlevel(abs(self.dataset.scale.x))

            # Store max zoom level in metadata
            self.rasterlayer.metadata.max_zoom = self.max_zoom
            self.rasterlayer.metadata.save()

            # Reduce max zoom by one if zoomdown flag was disabled
            if not self.zoomdown:
                self.max_zoom -= 1

            self.log("Started creating tiles", status=self.rasterlayer.parsestatus.CREATING_TILES)

            # Loop through all lower zoom levels and create tiles to
            # setup TMS aligned tiles in world mercator
            for iz in range(self.max_zoom + 1):
                self.create_tiles(iz)

            self.drop_empty_rasters()

            # Send signal for end of parsing
            rasterlayers_parser_ended.send(sender=self.rasterlayer.__class__, instance=self.rasterlayer)

            # Log success of parsing
            self.log("Successfully finished parsing raster", status=self.rasterlayer.parsestatus.FINISHED)
        except:
            self.log(traceback.format_exc(), status=self.rasterlayer.parsestatus.FAILED)
            raise
        finally:
            self.close_raster_file()
            shutil.rmtree(self.tmpdir)
Exemple #27
0
    def handle(self, *args, **options):
        options = Namespace(**options)

        if options.query_type == DOY_NAME:
            raster_query = self.build_query_doy(options)
            make_path = self.make_path_doy
            # TODO: finish this up for DOY queries
            #self.start_date = datetime.date(year=options.start_year, )
            self.interval = 'year'
        elif options.query_type == DATE_RANGE_NAME:
            raster_query = self.build_query_date_range(options)
            make_path = self.make_path_date_range
            self.start_date = options.start_date
            self.interval = 'day'
        else:
            raise CommandError(
                'Unrecognized query type: {}'.format(options.subparser_name),
            )

        with connection.cursor() as cursor:
            cursor.execute(PP_QUERY, [options.pourpoint_id])
            pp = cursor.fetchone()

            if not pp:
                raise CommandError('Pourpoint ID not found: {}'.format(
                    options.pourpoint_id,
                ))

            # TODO: override init and set values of all this crap I'm storing in the class
            self.path = make_path(options, pp[0])
            geotransform, wkt, nodata = pp[1], pp[2], pp[3]

            if os.path.isfile(self.path) and not options.force:
                return self.path

            cursor.execute(raster_query)

            if not cursor:
                raise CommandError('Query returned no raster data.')

            pathlib.Path(os.path.dirname(self.path)).mkdir(
                parents=True, exist_ok=True,
            )

            with nc.Dataset(
                self.path, 'w', clobber=True, format=NETCDF_FORMAT,
            ) as ds:
                ds = self._build_netcdf(ds, geotransform, wkt, nodata, cursor)

            raster_def = None
            for raster in cursor:
                raster = from_pgraster(raster[0])
                if not raster_def:
                    raster_def = raster
                else:
                    raster_def['bands'].extend(raster['bands'])

        # create the directory tree if any components missing


        # TODO: print with high verbosity level
        #for key in raster_def.keys():
        #    if key != 'bands':
        #        print(key, raster_def[key])

        raster = GDALRaster(raster_def)

        dsout =



        for index, band in enumerate(raster.bands):
            var[index] = band.data()

        return self.path
Exemple #28
0
    def test_raster_warp(self):
        # Create in memory raster
        source = GDALRaster({
            "datatype":
            1,
            "driver":
            "MEM",
            "name":
            "sourceraster",
            "width":
            4,
            "height":
            4,
            "nr_of_bands":
            1,
            "srid":
            3086,
            "origin": (500000, 400000),
            "scale": (100, -100),
            "skew": (0, 0),
            "bands": [{
                "data": range(16),
                "nodata_value": 255,
            }],
        })

        # Test altering the scale, width, and height of a raster
        data = {
            "scale": [200, -200],
            "width": 2,
            "height": 2,
        }
        target = source.warp(data)
        self.assertEqual(target.width, data["width"])
        self.assertEqual(target.height, data["height"])
        self.assertEqual(target.scale, data["scale"])
        self.assertEqual(target.bands[0].datatype(),
                         source.bands[0].datatype())
        self.assertEqual(target.name, "sourceraster_copy.MEM")
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(result, [5, 7, 13, 15])

        # Test altering the name and datatype (to float)
        data = {
            "name": "/path/to/targetraster.tif",
            "datatype": 6,
        }
        target = source.warp(data)
        self.assertEqual(target.bands[0].datatype(), 6)
        self.assertEqual(target.name, "/path/to/targetraster.tif")
        self.assertEqual(target.driver.name, "MEM")
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(
            result,
            [
                0.0,
                1.0,
                2.0,
                3.0,
                4.0,
                5.0,
                6.0,
                7.0,
                8.0,
                9.0,
                10.0,
                11.0,
                12.0,
                13.0,
                14.0,
                15.0,
            ],
        )
Exemple #29
0
 def setUp(self):
     self.rs_path = os.path.join(os.path.dirname(__file__), '../data/rasters/raster.tif')
     self.rs = GDALRaster(self.rs_path)
Exemple #30
0
    def test_raster_transform(self):
        tests = [
            3086,
            "3086",
            SpatialReference(3086),
        ]
        for srs in tests:
            with self.subTest(srs=srs):
                # Prepare tempfile and nodata value.
                rstfile = tempfile.NamedTemporaryFile(suffix=".tif")
                ndv = 99
                # Create in file based raster.
                source = GDALRaster({
                    "datatype":
                    1,
                    "driver":
                    "tif",
                    "name":
                    rstfile.name,
                    "width":
                    5,
                    "height":
                    5,
                    "nr_of_bands":
                    1,
                    "srid":
                    4326,
                    "origin": (-5, 5),
                    "scale": (2, -2),
                    "skew": (0, 0),
                    "bands": [{
                        "data": range(25),
                        "nodata_value": ndv,
                    }],
                })

                target = source.transform(srs)

                # Reload data from disk.
                target = GDALRaster(target.name)
                self.assertEqual(target.srs.srid, 3086)
                self.assertEqual(target.width, 7)
                self.assertEqual(target.height, 7)
                self.assertEqual(target.bands[0].datatype(),
                                 source.bands[0].datatype())
                self.assertAlmostEqual(target.origin[0], 9124842.791079799, 3)
                self.assertAlmostEqual(target.origin[1], 1589911.6476407414, 3)
                self.assertAlmostEqual(target.scale[0], 223824.82664250192, 3)
                self.assertAlmostEqual(target.scale[1], -223824.82664250192, 3)
                self.assertEqual(target.skew, [0, 0])

                result = target.bands[0].data()
                if numpy:
                    result = result.flatten().tolist()
                # The reprojection of a raster that spans over a large area
                # skews the data matrix and might introduce nodata values.
                self.assertEqual(
                    result,
                    [
                        ndv,
                        ndv,
                        ndv,
                        ndv,
                        4,
                        ndv,
                        ndv,
                        ndv,
                        ndv,
                        2,
                        3,
                        9,
                        ndv,
                        ndv,
                        ndv,
                        1,
                        2,
                        8,
                        13,
                        19,
                        ndv,
                        0,
                        6,
                        6,
                        12,
                        18,
                        18,
                        24,
                        ndv,
                        10,
                        11,
                        16,
                        22,
                        23,
                        ndv,
                        ndv,
                        ndv,
                        15,
                        21,
                        22,
                        ndv,
                        ndv,
                        ndv,
                        ndv,
                        20,
                        ndv,
                        ndv,
                        ndv,
                        ndv,
                    ],
                )
Exemple #31
0
 def test_vsi_invalid_buffer_error(self):
     msg = 'Failed creating VSI raster from the input buffer.'
     with self.assertRaisesMessage(GDALException, msg):
         GDALRaster(b'not-a-raster-buffer')
Exemple #32
0
    def get_rgb(self, data):
        # Get data arrays from tiles, by band if requested.
        for key, tile in data.items():

            keysplit = key.split(BAND_INDEX_SEPARATOR)
            variable = keysplit[0]

            if len(keysplit) > 1:
                band_index = int(keysplit[1])
            else:
                band_index = 0

            band = tile.bands[band_index]
            if variable == 'r':
                red = band.data()
                red_nodata = band.nodata_value
                red_gdal_dtype = band.datatype()
            elif variable == 'g':
                green = band.data()
                green_nodata = band.nodata_value
            elif variable == 'b':
                blue = band.data()
                blue_nodata = band.nodata_value

        # For tif requests, skip rgb rendering and return georeferenced tif file.
        if self.kwargs.get('frmt') == 'tif':
            vsi_path = os.path.join(VSI_FILESYSTEM_BASE_PATH, str(uuid.uuid4()))
            # Construct 3 band raster, assuming all
            ref = next(iter(data.values()))
            result = GDALRaster({
                'name': vsi_path,
                'driver': 'tif',
                'srid': WEB_MERCATOR_SRID,
                'datatype': red_gdal_dtype,
                'width': ref.width,
                'height': ref.height,
                'origin': ref.origin,
                'scale': ref.scale,
                'skew': ref.skew,
                'bands': [
                    {'data': red, 'nodata_value': red_nodata},
                    {'data': green.astype(red.dtype), 'nodata_value': green_nodata},
                    {'data': blue.astype(red.dtype), 'nodata_value': blue_nodata},
                ],
                'papsz_options': {
                    'compress': 'deflate',
                },
            })
            content_type = IMG_FORMATS['tif'][1]
            return HttpResponse(result.vsi_buffer, content_type)

        # Get scale for the image value range.
        if 'scale' in self.request.GET:
            # The scale is either a number or two numbers separated by comma.
            scale = self.request.GET.get('scale').split(',')
            if len(scale) == 1:
                scale_min = 0
                scale_max = float(scale[0])
            else:
                # Get min an max scale from
                scale_min = float(scale[0])
                scale_max = float(scale[1])

                # Clip the image minimum.
                red[red < scale_min] = scale_min
                green[green < scale_min] = scale_min
                blue[blue < scale_min] = scale_min

            # Clip the image maximum.
            red[red > scale_max] = scale_max
            green[green > scale_max] = scale_max
            blue[blue > scale_max] = scale_max

            # Scale the image.
            red = 255 * (red - scale_min) / scale_max
            green = 255 * (green - scale_min) / scale_max
            blue = 255 * (blue - scale_min) / scale_max

        if 'alpha' in self.request.GET:
            mode = 'RGBA'
            reshape = 4
            # Create the alpha channel.
            alpha = 255 * (red > 0) * (blue > 0) * (green > 0)
            img_array = numpy.array((red.ravel(), green.ravel(), blue.ravel(), alpha.ravel()))
        else:
            mode = 'RGB'
            reshape = 3
            img_array = numpy.array((red.ravel(), green.ravel(), blue.ravel()))

        # Reshape array into tile size.
        img_array = img_array.T.reshape(WEB_MERCATOR_TILESIZE, WEB_MERCATOR_TILESIZE, reshape).astype('uint8')

        # Create image from array
        img = Image.fromarray(img_array, mode=mode)
        stats = {}

        # Return rendered image
        return self.write_img_to_response(img, stats)
Exemple #33
0
 def setUp(self):
     self.rs_path = os.path.join(os.path.dirname(__file__), '../data/rasters/raster.tif')
     rs = GDALRaster(self.rs_path)
     self.band = rs.bands[0]
Exemple #34
0
    def test_gdal_functions(self):
        raster = GDALRaster(os.path.join("app", "fixtures", "orthophoto.tif"))

        self.assertTrue(raster.srid == 32615)
        self.assertTrue(raster.width == 212)
Exemple #35
0
    def test_all_gis_lookups_with_rasters(self):
        """
        Evaluate all possible lookups for all input combinations (i.e.
        raster-raster, raster-geom, geom-raster) and for projected and
        unprojected coordinate systems. This test just checks that the lookup
        can be called, but doesn't check if the result makes logical sense.
        """
        from django.contrib.gis.db.backends.postgis.operations import PostGISOperations

        # Create test raster and geom.
        rast = GDALRaster(json.loads(JSON_RASTER))
        stx_pnt = GEOSGeometry("POINT (-95.370401017314293 29.704867409475465)", 4326)
        stx_pnt.transform(3086)

        lookups = [
            (name, lookup)
            for name, lookup in BaseSpatialField.get_lookups().items()
            if issubclass(lookup, GISLookup)
        ]
        self.assertNotEqual(lookups, [], "No lookups found")
        # Loop through all the GIS lookups.
        for name, lookup in lookups:
            # Construct lookup filter strings.
            combo_keys = [
                field + name
                for field in [
                    "rast__",
                    "rast__",
                    "rastprojected__0__",
                    "rast__",
                    "rastprojected__",
                    "geom__",
                    "rast__",
                ]
            ]
            if issubclass(lookup, DistanceLookupBase):
                # Set lookup values for distance lookups.
                combo_values = [
                    (rast, 50, "spheroid"),
                    (rast, 0, 50, "spheroid"),
                    (rast, 0, D(km=1)),
                    (stx_pnt, 0, 500),
                    (stx_pnt, D(km=1000)),
                    (rast, 500),
                    (json.loads(JSON_RASTER), 500),
                ]
            elif name == "relate":
                # Set lookup values for the relate lookup.
                combo_values = [
                    (rast, "T*T***FF*"),
                    (rast, 0, "T*T***FF*"),
                    (rast, 0, "T*T***FF*"),
                    (stx_pnt, 0, "T*T***FF*"),
                    (stx_pnt, "T*T***FF*"),
                    (rast, "T*T***FF*"),
                    (json.loads(JSON_RASTER), "T*T***FF*"),
                ]
            elif name == "isvalid":
                # The isvalid lookup doesn't make sense for rasters.
                continue
            elif PostGISOperations.gis_operators[name].func:
                # Set lookup values for all function based operators.
                combo_values = [
                    rast,
                    (rast, 0),
                    (rast, 0),
                    (stx_pnt, 0),
                    stx_pnt,
                    rast,
                    json.loads(JSON_RASTER),
                ]
            else:
                # Override band lookup for these, as it's not supported.
                combo_keys[2] = "rastprojected__" + name
                # Set lookup values for all other operators.
                combo_values = [
                    rast,
                    None,
                    rast,
                    stx_pnt,
                    stx_pnt,
                    rast,
                    json.loads(JSON_RASTER),
                ]

            # Create query filter combinations.
            self.assertEqual(
                len(combo_keys),
                len(combo_values),
                "Number of lookup names and values should be the same",
            )
            combos = [x for x in zip(combo_keys, combo_values) if x[1]]
            self.assertEqual(
                [(n, x) for n, x in enumerate(combos) if x in combos[:n]],
                [],
                "There are repeated test lookups",
            )
            combos = [{k: v} for k, v in combos]

            for combo in combos:
                # Apply this query filter.
                qs = RasterModel.objects.filter(**combo)

                # Evaluate normal filter qs.
                self.assertIn(qs.count(), [0, 1])

            # Evaluate on conditional Q expressions.
            qs = RasterModel.objects.filter(Q(**combos[0]) & Q(**combos[1]))
            self.assertIn(qs.count(), [0, 1])
Exemple #36
0
    def test_raster_transform(self):
        if GDAL_VERSION < (1, 8, 1):
            self.skipTest("GDAL >= 1.8.1 is required for this test")
        # Prepare tempfile and nodata value
        rstfile = tempfile.NamedTemporaryFile(suffix=".tif")
        ndv = 99

        # Create in file based raster
        source = GDALRaster(
            {
                "datatype": 1,
                "driver": "tif",
                "name": rstfile.name,
                "width": 5,
                "height": 5,
                "nr_of_bands": 1,
                "srid": 4326,
                "origin": (-5, 5),
                "scale": (2, -2),
                "skew": (0, 0),
                "bands": [{"data": range(25), "nodata_value": ndv}],
            }
        )

        # Transform raster into srid 4326.
        target = source.transform(3086)

        # Reload data from disk
        target = GDALRaster(target.name)

        self.assertEqual(target.srs.srid, 3086)
        self.assertEqual(target.width, 7)
        self.assertEqual(target.height, 7)
        self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype())
        self.assertEqual(target.origin, [9124842.791079799, 1589911.6476407414])
        self.assertEqual(target.scale, [223824.82664250192, -223824.82664250192])
        self.assertEqual(target.skew, [0, 0])

        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # The reprojection of a raster that spans over a large area
        # skews the data matrix and might introduce nodata values.
        self.assertEqual(
            result,
            [
                ndv,
                ndv,
                ndv,
                ndv,
                4,
                ndv,
                ndv,
                ndv,
                ndv,
                2,
                3,
                9,
                ndv,
                ndv,
                ndv,
                1,
                2,
                8,
                13,
                19,
                ndv,
                0,
                6,
                6,
                12,
                18,
                18,
                24,
                ndv,
                10,
                11,
                16,
                22,
                23,
                ndv,
                ndv,
                ndv,
                15,
                21,
                22,
                ndv,
                ndv,
                ndv,
                ndv,
                20,
                ndv,
                ndv,
                ndv,
                ndv,
            ],
        )
Exemple #37
0
 def test_lookup_input_tuple_too_long(self):
     rast = GDALRaster(json.loads(JSON_RASTER))
     msg = "Tuple too long for lookup bbcontains."
     with self.assertRaisesMessage(ValueError, msg):
         RasterModel.objects.filter(rast__bbcontains=(rast, 1, 2))
Exemple #38
0
def __make_rastertiles_Z__(src_dataset: rio.DatasetReader, world_size: float,
                           tile_size: int, zoom: int) -> list():

    # get bands
    src_bands = src_dataset.read()

    # structure for store tiles
    tiles = []

    # get bounds
    src_bbox = src_dataset.bounds
    src_bbox = [src_bbox.left, src_bbox.top, src_bbox.right, src_bbox.bottom]

    # get pixel size
    pixel_size = __pixel_size__(world_size, tile_size, zoom)

    # get all quadrant
    quadrants = __make_quadrants__(src_bbox, zoom, world_size, 1)

    for xmin, ymin, xmax, ymax in quadrants:

        # get bbox of quadrant
        Xmin, Ymin, Xmax, Ymax = list(
            __tile_world_bbox__(xmin, ymin, zoom, world_size, tile_size))

        # get pixel size
        pixel_size = __pixel_size__(world_size, tile_size, zoom)

        # make dst shape (3, tsize, tsize), 3 is fix because it's an image RGB
        dst_shape = (3, tile_size, tile_size)

        # make transform with orig (Xmin, Ymin) and scale (psize, -psize)
        dst_transform = A.translation(Xmin, Ymin) * A.scale(
            pixel_size, -pixel_size)

        dtype = src_dataset.dtypes[0]

        if dtype == rio.uint8:
            datatype = 1
        elif dtype == rio.uint16:
            datatype = 2
        elif dtype == rio.int16:
            datatype = 3
        elif dtype == rio.uint32:
            datatype = 4
        elif dtype == rio.int32:
            datatype = 5
        elif dtype == rio.float32:
            datatype = 6
        elif dtype == rio.float64:
            datatype = 7
        else:
            assert False

        # init dst bands
        dst_bands = np.zeros(dst_shape, dtype=dtype)

        count = dst_bands.shape[0]
        nodata = 0 if src_dataset.nodata is None else src_dataset.nodata

        # make reprojection for each bands
        for i in range(count):

            try:

                reproject(source=src_bands[i],
                          destination=dst_bands[i],
                          src_transform=src_dataset.transform,
                          src_crs=src_dataset.crs,
                          src_nodata=nodata,
                          dst_transform=dst_transform,
                          dst_crs=src_dataset.crs)

            except IndexError:
                continue

        gdal_bands = [{
            'data': dst_bands[x],
            'nodata_value': nodata
        } for x in range(count)]

        gdal_raster = GDALRaster({
            'srid': WEB_MERCATOR_SRID,
            'width': tile_size,
            'height': tile_size,
            'datatype': datatype,
            'nr_of_bands': count,
            'origin': [Xmin, Ymin],
            'scale': [pixel_size, -pixel_size],
            'bands': gdal_bands
        })

        tiles.append((zoom, xmin, ymin, gdal_raster))

    del src_bands

    # return structure
    return tiles
Exemple #39
0
 def create_model_object_raster(self, view_resource, row):
     obj_model = view_resource.model_class()()
     setattr(obj_model, view_resource.pk_name(), row[0])
     rst = GDALRaster(row[1].tobytes())
     setattr(obj_model, view_resource.spatial_field_name(), rst)
     return obj_model
Exemple #40
0
    def extract_assets_and_complete(self):
        """
        Extracts assets/all.zip, populates task fields where required and assure COGs
        It will raise a zipfile.BadZipFile exception is the archive is corrupted.
        :return:
        """
        assets_dir = self.assets_path("")
        zip_path = self.assets_path("all.zip")

        # Extract from zip
        with zipfile.ZipFile(zip_path, "r") as zip_h:
            zip_h.extractall(assets_dir)

        logger.info("Extracted all.zip for {}".format(self))

        # Populate *_extent fields
        extent_fields = [
            (os.path.realpath(
                self.assets_path("odm_orthophoto",
                                 "odm_orthophoto.tif")), 'orthophoto_extent'),
            (os.path.realpath(self.assets_path("odm_dem",
                                               "dsm.tif")), 'dsm_extent'),
            (os.path.realpath(self.assets_path("odm_dem",
                                               "dtm.tif")), 'dtm_extent'),
        ]

        for raster_path, field in extent_fields:
            if os.path.exists(raster_path):
                # Make sure this is a Cloud Optimized GeoTIFF
                # if not, it will be created
                try:
                    assure_cogeo(raster_path)
                except IOError as e:
                    logger.warning(
                        "Cannot create Cloud Optimized GeoTIFF for %s (%s). This will result in degraded visualization performance."
                        % (raster_path, str(e)))

                # Read extent and SRID
                raster = GDALRaster(raster_path)
                extent = OGRGeometry.from_bbox(raster.extent)

                # Make sure PostGIS supports it
                with connection.cursor() as cursor:
                    cursor.execute(
                        "SELECT SRID FROM spatial_ref_sys WHERE SRID = %s",
                        [raster.srid])
                    if cursor.rowcount == 0:
                        raise NodeServerError(
                            gettext(
                                "Unsupported SRS %(code)s. Please make sure you picked a supported SRS."
                            ) % {'code': str(raster.srid)})

                # It will be implicitly transformed into the SRID of the model’s field
                # self.field = GEOSGeometry(...)
                setattr(self, field, GEOSGeometry(extent.wkt,
                                                  srid=raster.srid))

                logger.info("Populated extent field with {} for {}".format(
                    raster_path, self))

        self.update_available_assets_field()
        self.running_progress = 1.0
        self.console_output += gettext("Done!") + "\n"
        self.status = status_codes.COMPLETED
        self.save()

        from app.plugins import signals as plugin_signals
        plugin_signals.task_completed.send_robust(sender=self.__class__,
                                                  task_id=self.id)
Exemple #41
0
    def process(self):
        """
        This method contains the logic for processing tasks asynchronously
        from a background thread or from a worker. Here tasks that are
        ready to be processed execute some logic. This could be communication
        with a processing node or executing a pending action.
        """

        try:
            if self.pending_action == pending_actions.RESIZE:
                resized_images = self.resize_images()
                self.resize_gcp(resized_images)
                self.pending_action = None
                self.save()

            if self.auto_processing_node and not self.status in [
                    status_codes.FAILED, status_codes.CANCELED
            ]:
                # No processing node assigned and need to auto assign
                if self.processing_node is None:
                    # Assign first online node with lowest queue count
                    self.processing_node = ProcessingNode.find_best_available_node(
                    )
                    if self.processing_node:
                        self.processing_node.queue_count += 1  # Doesn't have to be accurate, it will get overridden later
                        self.processing_node.save()

                        logger.info(
                            "Automatically assigned processing node {} to {}".
                            format(self.processing_node, self))
                        self.save()

                # Processing node assigned, but is offline and no errors
                if self.processing_node and not self.processing_node.is_online(
                ):
                    # If we are queued up
                    # detach processing node, and reassignment
                    # will be processed at the next tick
                    if self.status == status_codes.QUEUED:
                        logger.info(
                            "Processing node {} went offline, reassigning {}..."
                            .format(self.processing_node, self))
                        self.uuid = ''
                        self.processing_node = None
                        self.status = None
                        self.save()

                    elif self.status == status_codes.RUNNING:
                        # Task was running and processing node went offline
                        # It could have crashed due to low memory
                        # or perhaps it went offline due to network errors.
                        # We can't easily differentiate between the two, so we need
                        # to notify the user because if it crashed due to low memory
                        # the user might need to take action (or be stuck in an infinite loop)
                        raise ProcessingError(
                            "Processing node went offline. This could be due to insufficient memory or a network error."
                        )

            if self.processing_node:
                # Need to process some images (UUID not yet set and task doesn't have pending actions)?
                if not self.uuid and self.pending_action is None and self.status is None:
                    logger.info("Processing... {}".format(self))

                    images = [
                        image.path() for image in self.imageupload_set.all()
                    ]

                    # This takes a while
                    uuid = self.processing_node.process_new_task(
                        images, self.name, self.options)

                    # Refresh task object before committing change
                    self.refresh_from_db()
                    self.uuid = uuid
                    self.save()

                    # TODO: log process has started processing

            if self.pending_action is not None:
                if self.pending_action == pending_actions.CANCEL:
                    # Do we need to cancel the task on the processing node?
                    logger.info("Canceling {}".format(self))
                    if self.processing_node and self.uuid:
                        # Attempt to cancel the task on the processing node
                        # We don't care if this fails (we tried)
                        try:
                            self.processing_node.cancel_task(self.uuid)
                        except ProcessingException:
                            logger.warning(
                                "Could not cancel {} on processing node. We'll proceed anyway..."
                                .format(self))

                        self.status = status_codes.CANCELED
                        self.pending_action = None
                        self.save()
                    else:
                        raise ProcessingError(
                            "Cannot cancel a task that has no processing node or UUID"
                        )

                elif self.pending_action == pending_actions.RESTART:
                    logger.info("Restarting {}".format(self))
                    if self.processing_node:

                        # Check if the UUID is still valid, as processing nodes purge
                        # results after a set amount of time, the UUID might have been eliminated.
                        uuid_still_exists = False

                        if self.uuid:
                            try:
                                info = self.processing_node.get_task_info(
                                    self.uuid)
                                uuid_still_exists = info['uuid'] == self.uuid
                            except ProcessingException:
                                pass

                        need_to_reprocess = False

                        if uuid_still_exists:
                            # Good to go
                            try:
                                self.processing_node.restart_task(
                                    self.uuid, self.options)
                            except ProcessingError as e:
                                # Something went wrong
                                logger.warning(
                                    "Could not restart {}, will start a new one"
                                    .format(self))
                                need_to_reprocess = True
                        else:
                            need_to_reprocess = True

                        if need_to_reprocess:
                            logger.info(
                                "{} needs to be reprocessed".format(self))

                            # Task has been purged (or processing node is offline)
                            # Process this as a new task
                            # Removing its UUID will cause the scheduler
                            # to process this the next tick
                            self.uuid = ''

                            # We also remove the "rerun-from" parameter if it's set
                            self.options = list(
                                filter(lambda d: d['name'] != 'rerun-from',
                                       self.options))

                        self.console_output = ""
                        self.processing_time = -1
                        self.status = None
                        self.last_error = None
                        self.pending_action = None
                        self.save()
                    else:
                        raise ProcessingError(
                            "Cannot restart a task that has no processing node"
                        )

                elif self.pending_action == pending_actions.REMOVE:
                    logger.info("Removing {}".format(self))
                    if self.processing_node and self.uuid:
                        # Attempt to delete the resources on the processing node
                        # We don't care if this fails, as resources on processing nodes
                        # Are expected to be purged on their own after a set amount of time anyway
                        try:
                            self.processing_node.remove_task(self.uuid)
                        except ProcessingException:
                            pass

                    # What's more important is that we delete our task properly here
                    self.delete()

                    # Stop right here!
                    return

            if self.processing_node:
                # Need to update status (first time, queued or running?)
                if self.uuid and self.status in [
                        None, status_codes.QUEUED, status_codes.RUNNING
                ]:
                    # Update task info from processing node
                    info = self.processing_node.get_task_info(self.uuid)

                    self.processing_time = info["processingTime"]
                    self.status = info["status"]["code"]

                    current_lines_count = len(self.console_output.split("\n"))
                    console_output = self.processing_node.get_task_console_output(
                        self.uuid, current_lines_count)
                    if len(console_output) > 0:
                        self.console_output += console_output + '\n'

                    if "errorMessage" in info["status"]:
                        self.last_error = info["status"]["errorMessage"]

                    # Has the task just been canceled, failed, or completed?
                    if self.status in [
                            status_codes.FAILED, status_codes.COMPLETED,
                            status_codes.CANCELED
                    ]:
                        logger.info("Processing status: {} for {}".format(
                            self.status, self))

                        if self.status == status_codes.COMPLETED:
                            assets_dir = self.assets_path("")

                            # Remove previous assets directory
                            if os.path.exists(assets_dir):
                                logger.info(
                                    "Removing old assets directory: {} for {}".
                                    format(assets_dir, self))
                                shutil.rmtree(assets_dir)

                            os.makedirs(assets_dir)

                            logger.info(
                                "Downloading all.zip for {}".format(self))

                            # Download all assets
                            zip_stream = self.processing_node.download_task_asset(
                                self.uuid, "all.zip")
                            zip_path = os.path.join(assets_dir, "all.zip")
                            with open(zip_path, 'wb') as fd:
                                for chunk in zip_stream.iter_content(4096):
                                    fd.write(chunk)

                            logger.info(
                                "Done downloading all.zip for {}".format(self))

                            # Extract from zip
                            with zipfile.ZipFile(zip_path, "r") as zip_h:
                                zip_h.extractall(assets_dir)

                            logger.info(
                                "Extracted all.zip for {}".format(self))

                            # Populate *_extent fields
                            extent_fields = [
                                (os.path.realpath(
                                    self.assets_path("odm_orthophoto",
                                                     "odm_orthophoto.tif")),
                                 'orthophoto_extent'),
                                (os.path.realpath(
                                    self.assets_path("odm_dem", "dsm.tif")),
                                 'dsm_extent'),
                                (os.path.realpath(
                                    self.assets_path("odm_dem", "dtm.tif")),
                                 'dtm_extent'),
                                (os.path.realpath(
                                    self.assets_path("odm_dem",
                                                     "regions.tif")),
                                 'regions_extent'),
                            ]

                            for raster_path, field in extent_fields:
                                if os.path.exists(raster_path):
                                    # Read extent and SRID
                                    raster = GDALRaster(raster_path)
                                    extent = OGRGeometry.from_bbox(
                                        raster.extent)

                                    # It will be implicitly transformed into the SRID of the model’s field
                                    # self.field = GEOSGeometry(...)
                                    setattr(
                                        self, field,
                                        GEOSGeometry(extent.wkt,
                                                     srid=raster.srid))

                                    logger.info(
                                        "Populated extent field with {} for {}"
                                        .format(raster_path, self))

                            self.update_available_assets_field()
                            self.save()

                            from app.plugins import signals as plugin_signals
                            plugin_signals.task_completed.send_robust(
                                sender=self.__class__, task_id=self.id)
                        else:
                            # FAILED, CANCELED
                            self.save()
                    else:
                        # Still waiting...
                        self.save()

        except ProcessingError as e:
            self.set_failure(str(e))
        except (ConnectionRefusedError, ConnectionError) as e:
            logger.warning(
                "{} cannot communicate with processing node: {}".format(
                    self, str(e)))
        except ProcessingTimeout as e:
            logger.warning(
                "{} timed out with error: {}. We'll try reprocessing at the next tick."
                .format(self, str(e)))
    def open_raster_file(self):
        """
        Get raster source file to extract tiles from.

        This makes a local copy of rasterfile, unzips the raster and reprojects
        it into web mercator if necessary. The reprojected raster is stored for
        reuse such that reprojection does only happen once.

        The local copy of the raster is needed if files are stored on remote
        storages.
        """
        reproj, created = RasterLayerReprojected.objects.get_or_create(rasterlayer=self.rasterlayer)
        # Check if the raster has already been reprojected
        has_reprojected = reproj.rasterfile.name not in (None, '')

        # Create workdir
        raster_workdir = getattr(settings, 'RASTER_WORKDIR', None)
        self.tmpdir = tempfile.mkdtemp(dir=raster_workdir)

        # Choose source for raster data, use the reprojected version if it exists.
        if self.rasterlayer.source_url and not has_reprojected:
            url_path = urlparse(self.rasterlayer.source_url).path
            filename = url_path.split('/')[-1]
            filepath = os.path.join(self.tmpdir, filename)
            urlretrieve(self.rasterlayer.source_url, filepath)
        else:
            if has_reprojected:
                rasterfile_source = reproj.rasterfile
            else:
                rasterfile_source = self.rasterlayer.rasterfile

            if not rasterfile_source.name:
                raise RasterException('No data source found. Provide a rasterfile or a source url.')

            # Copy raster file source to local folder
            filepath = os.path.join(self.tmpdir, os.path.basename(rasterfile_source.name))
            rasterfile = open(filepath, 'wb')
            for chunk in rasterfile_source.chunks():
                rasterfile.write(chunk)
            rasterfile.close()

        # If the raster file is compressed, decompress it, otherwise try to
        # open the source file directly.
        if os.path.splitext(filepath)[1].lower() == '.zip':
            # Open and extract zipfile
            zf = zipfile.ZipFile(filepath)
            zf.extractall(self.tmpdir)

            # Remove zipfile
            os.remove(filepath)

            # Get filelist from directory
            matches = []
            for root, dirnames, filenames in os.walk(self.tmpdir):
                for filename in fnmatch.filter(filenames, '*.*'):
                    matches.append(os.path.join(root, filename))

            # Open the first raster file found in the matched files.
            self.dataset = None
            for match in matches:
                try:
                    self.dataset = GDALRaster(match)
                    break
                except GDALException:
                    pass

            # Raise exception if no file could be opened by gdal.
            if not self.dataset:
                raise RasterException('Could not open rasterfile.')
        else:
            self.dataset = GDALRaster(filepath)

        # Override srid if provided
        if self.rasterlayer.srid:
            try:
                self.dataset = GDALRaster(self.dataset.name, write=True)
            except GDALException:
                raise RasterException(
                    'Could not override srid because the driver for this '
                    'type of raster does not support write mode.'
                )
            self.dataset.srs = self.rasterlayer.srid
Exemple #43
0
 def test_nonexistent_file(self):
     msg = 'Unable to read raster source input "nonexistent.tif".'
     with self.assertRaisesMessage(GDALException, msg):
         GDALRaster('nonexistent.tif')
Exemple #44
0
    def process(self):
        """
        This method contains the logic for processing tasks asynchronously
        from a background thread or from the scheduler. Here tasks that are
        ready to be processed execute some logic. This could be communication
        with a processing node or executing a pending action.
        """

        try:
            if self.auto_processing_node and not self.status in [status_codes.FAILED, status_codes.CANCELED]:
                # No processing node assigned and need to auto assign
                if self.processing_node is None:
                    # Assign first online node with lowest queue count
                    self.processing_node = ProcessingNode.find_best_available_node()
                    if self.processing_node:
                        self.processing_node.queue_count += 1 # Doesn't have to be accurate, it will get overriden later
                        self.processing_node.save()

                        logger.info("Automatically assigned processing node {} to {}".format(self.processing_node, self))
                        self.save()

                # Processing node assigned, but is offline and no errors
                if self.processing_node and not self.processing_node.is_online():
                    # Detach processing node, will be processed at the next tick
                    logger.info("Processing node {} went offline, reassigning {}...".format(self.processing_node, self))
                    self.uuid = ''
                    self.processing_node = None
                    self.save()

            if self.processing_node:
                # Need to process some images (UUID not yet set and task doesn't have pending actions)?
                if not self.uuid and self.pending_action is None and self.status is None:
                    logger.info("Processing... {}".format(self))

                    images = [image.path() for image in self.imageupload_set.all()]

                    # This takes a while
                    uuid = self.processing_node.process_new_task(images, self.name, self.options)

                    # Refresh task object before committing change
                    self.refresh_from_db()
                    self.uuid = uuid
                    self.save()

                    # TODO: log process has started processing

            if self.pending_action is not None:
                if self.pending_action == pending_actions.CANCEL:
                    # Do we need to cancel the task on the processing node?
                    logger.info("Canceling {}".format(self))
                    if self.processing_node and self.uuid:
                        # Attempt to cancel the task on the processing node
                        # We don't care if this fails (we tried)
                        try:
                            self.processing_node.cancel_task(self.uuid)
                            self.status = None
                        except ProcessingException:
                            logger.warning("Could not cancel {} on processing node. We'll proceed anyway...".format(self))
                            self.status = status_codes.CANCELED

                        self.pending_action = None
                        self.save()
                    else:
                        raise ProcessingError("Cannot cancel a task that has no processing node or UUID")

                elif self.pending_action == pending_actions.RESTART:
                    logger.info("Restarting {}".format(self))
                    if self.processing_node:

                        # Check if the UUID is still valid, as processing nodes purge
                        # results after a set amount of time, the UUID might have eliminated.
                        uuid_still_exists = False

                        if self.uuid:
                            try:
                                info = self.processing_node.get_task_info(self.uuid)
                                uuid_still_exists = info['uuid'] == self.uuid
                            except ProcessingException:
                                pass

                        if uuid_still_exists:
                            # Good to go
                            try:
                                self.processing_node.restart_task(self.uuid)
                            except ProcessingError as e:
                                # Something went wrong
                                logger.warning("Could not restart {}, will start a new one".format(self))
                                self.uuid = ''
                        else:
                            # Task has been purged (or processing node is offline)
                            # Process this as a new task
                            # Removing its UUID will cause the scheduler
                            # to process this the next tick
                            self.uuid = ''

                        self.console_output = ""
                        self.processing_time = -1
                        self.status = None
                        self.last_error = None
                        self.pending_action = None
                        self.save()
                    else:
                        raise ProcessingError("Cannot restart a task that has no processing node")

                elif self.pending_action == pending_actions.REMOVE:
                    logger.info("Removing {}".format(self))
                    if self.processing_node and self.uuid:
                        # Attempt to delete the resources on the processing node
                        # We don't care if this fails, as resources on processing nodes
                        # Are expected to be purged on their own after a set amount of time anyway
                        try:
                            self.processing_node.remove_task(self.uuid)
                        except ProcessingException:
                            pass

                    # What's more important is that we delete our task properly here
                    self.delete()

                    # Stop right here!
                    return

            if self.processing_node:
                # Need to update status (first time, queued or running?)
                if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]:
                    # Update task info from processing node
                    info = self.processing_node.get_task_info(self.uuid)

                    self.processing_time = info["processingTime"]
                    self.status = info["status"]["code"]

                    current_lines_count = len(self.console_output.split("\n")) - 1
                    self.console_output += self.processing_node.get_task_console_output(self.uuid, current_lines_count)

                    if "errorMessage" in info["status"]:
                        self.last_error = info["status"]["errorMessage"]

                    # Has the task just been canceled, failed, or completed?
                    if self.status in [status_codes.FAILED, status_codes.COMPLETED, status_codes.CANCELED]:
                        logger.info("Processing status: {} for {}".format(self.status, self))

                        if self.status == status_codes.COMPLETED:
                            assets_dir = self.assets_path("")
                            if not os.path.exists(assets_dir):
                                os.makedirs(assets_dir)

                            logger.info("Downloading all.zip for {}".format(self))

                            # Download all assets
                            zip_stream = self.processing_node.download_task_asset(self.uuid, "all.zip")
                            zip_path = os.path.join(assets_dir, "all.zip")
                            with open(zip_path, 'wb') as fd:
                                for chunk in zip_stream.iter_content(4096):
                                    fd.write(chunk)

                            logger.info("Done downloading all.zip for {}".format(self))

                            # Extract from zip
                            with zipfile.ZipFile(zip_path, "r") as zip_h:
                                zip_h.extractall(assets_dir)

                            logger.info("Extracted all.zip for {}".format(self))

                            # Add to database orthophoto
                            orthophoto_path = os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif"))
                            if os.path.exists(orthophoto_path):
                                orthophoto = GDALRaster(orthophoto_path, write=True)

                                # We need to transform to 4326 before we can store it
                                # as an offdb raster field
                                orthophoto_4326_path = os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto_4326.tif"))
                                self.orthophoto = orthophoto.transform(4326, 'GTiff', orthophoto_4326_path)

                                logger.info("Imported orthophoto {} for {}".format(orthophoto_4326_path, self))

                            # Remove old odm_texturing.zip archive (if any)
                            textured_model_archive = self.assets_path(self.get_textured_model_filename())
                            if os.path.exists(textured_model_archive):
                                os.remove(textured_model_archive)

                            self.save()
                        else:
                            # FAILED, CANCELED
                            self.save()
                    else:
                        # Still waiting...
                        self.save()

        except ProcessingError as e:
            self.set_failure(str(e))
        except (ConnectionRefusedError, ConnectionError) as e:
            logger.warning("{} cannot communicate with processing node: {}".format(self, str(e)))
        except ProcessingTimeout as e:
            logger.warning("{} timed out with error: {}. We'll try reprocessing at the next tick.".format(self, str(e)))
Exemple #45
0
class RasterLayerParser(object):
    """
    Class to parse raster layers.
    """
    def __init__(self, rasterlayer_id):
        self.rasterlayer = RasterLayer.objects.get(id=rasterlayer_id)

        # Set raster tilesize
        self.tilesize = int(getattr(settings, 'RASTER_TILESIZE', WEB_MERCATOR_TILESIZE))
        self.batch_step_size = int(getattr(settings, 'RASTER_BATCH_STEP_SIZE', BATCH_STEP_SIZE))

    def log(self, msg, status=None, zoom=None):
        """
        Write a message to the parse log of the rasterlayer instance and update
        the parse status object.
        """
        parsestatus = self.rasterlayer.parsestatus
        parsestatus.refresh_from_db()

        if status is not None:
            parsestatus.status = status

        if zoom is not None and zoom not in parsestatus.tile_levels:
            parsestatus.tile_levels.append(zoom)
            parsestatus.tile_levels.sort()

        # Prepare datetime stamp for log
        now = '[{0}] '.format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))

        if parsestatus.log:
            now = '\n' + now

        parsestatus.log += now + msg
        parsestatus.save()

    def open_raster_file(self):
        """
        Get raster source file to extract tiles from.

        This makes a local copy of rasterfile, unzips the raster and reprojects
        it into web mercator if necessary. The reprojected raster is stored for
        reuse such that reprojection does only happen once.

        The local copy of the raster is needed if files are stored on remote
        storages.
        """
        reproj, created = RasterLayerReprojected.objects.get_or_create(rasterlayer=self.rasterlayer)
        # Check if the raster has already been reprojected
        has_reprojected = reproj.rasterfile.name not in (None, '')

        # Create workdir
        raster_workdir = getattr(settings, 'RASTER_WORKDIR', None)
        self.tmpdir = tempfile.mkdtemp(dir=raster_workdir)

        # Choose source for raster data, use the reprojected version if it exists.
        if self.rasterlayer.source_url and not has_reprojected:
            url = self.rasterlayer.source_url
            if url.lower().startswith('http') or url.startswith('file'):
                url_path = urlparse(self.rasterlayer.source_url).path
                filename = url_path.split('/')[-1]
                filepath = os.path.join(self.tmpdir, filename)
                urlretrieve(self.rasterlayer.source_url, filepath)
            elif url.startswith('s3'):
                # Get the bucket name and file key, assuming the following url
                # strucure: s3://BUCKET_NAME/BUCKET_KEY
                bucket_name = url.split('s3://')[1].split('/')[0]
                bucket_key = '/'.join(url.split('s3://')[1].split('/')[1:])
                # Assume the file name is the last piece of the key.
                filename = bucket_key.split('/')[-1]
                filepath = os.path.join(self.tmpdir, filename)
                # Get file from s3.
                s3 = boto3.resource('s3')
                bucket = s3.Bucket(bucket_name)
                bucket.download_file(bucket_key, filepath, ExtraArgs={'RequestPayer': 'requester'})
            else:
                raise RasterException('Only http(s) and s3 urls are supported.')
        else:
            if has_reprojected:
                rasterfile_source = reproj.rasterfile
            else:
                rasterfile_source = self.rasterlayer.rasterfile

            if not rasterfile_source.name:
                raise RasterException('No data source found. Provide a rasterfile or a source url.')

            # Copy raster file source to local folder
            filepath = os.path.join(self.tmpdir, os.path.basename(rasterfile_source.name))
            rasterfile = open(filepath, 'wb')
            for chunk in rasterfile_source.chunks():
                rasterfile.write(chunk)
            rasterfile.close()

        # If the raster file is compressed, decompress it, otherwise try to
        # open the source file directly.
        if os.path.splitext(filepath)[1].lower() == '.zip':
            # Open and extract zipfile
            zf = zipfile.ZipFile(filepath)
            zf.extractall(self.tmpdir)

            # Remove zipfile
            os.remove(filepath)

            # Get filelist from directory
            matches = []
            for root, dirnames, filenames in os.walk(self.tmpdir):
                for filename in fnmatch.filter(filenames, '*.*'):
                    matches.append(os.path.join(root, filename))

            # Open the first raster file found in the matched files.
            self.dataset = None
            for match in matches:
                try:
                    self.dataset = GDALRaster(match)
                    break
                except GDALException:
                    pass

            # Raise exception if no file could be opened by gdal.
            if not self.dataset:
                raise RasterException('Could not open rasterfile.')
        else:
            self.dataset = GDALRaster(filepath)

        # Override srid if provided
        if self.rasterlayer.srid:
            try:
                self.dataset = GDALRaster(self.dataset.name, write=True)
            except GDALException:
                raise RasterException(
                    'Could not override srid because the driver for this '
                    'type of raster does not support write mode.'
                )
            self.dataset.srs = self.rasterlayer.srid

    def reproject_rasterfile(self):
        """
        Reproject the rasterfile into web mercator.
        """
        # Return if reprojected rasterfile already exists.
        if hasattr(self.rasterlayer, 'reprojected') and self.rasterlayer.reprojected.rasterfile.name:
            return

        # Return if the raster already has the right projection
        # and nodata value is acceptable.
        if self.dataset.srs.srid == WEB_MERCATOR_SRID:
            # SRID was not manually specified.
            if self.rasterlayer.nodata in ('', None):
                return
            # All bands from dataset already have the same nodata value as the
            # one that was manually specified.
            if all([self.rasterlayer.nodata == band.nodata_value
                    for band in self.dataset.bands]):
                return
        else:
            # Log projection change if original raster is not in web mercator.
            self.log(
                'Transforming raster to SRID {0}'.format(WEB_MERCATOR_SRID),
                status=self.rasterlayer.parsestatus.REPROJECTING_RASTER,
            )

        # Reproject the dataset.
        self.dataset = self.dataset.transform(
            WEB_MERCATOR_SRID,
            driver=INTERMEDIATE_RASTER_FORMAT,
        )

        # Manually override nodata value if neccessary
        if self.rasterlayer.nodata not in ('', None):
            self.log(
                'Setting no data values to {0}.'.format(self.rasterlayer.nodata),
                status=self.rasterlayer.parsestatus.REPROJECTING_RASTER,
            )
            for band in self.dataset.bands:
                band.nodata_value = float(self.rasterlayer.nodata)

        # Compress reprojected raster file and store it
        if self.rasterlayer.store_reprojected:
            dest = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.zip')
            dest_zip = zipfile.ZipFile(dest.name, 'w', allowZip64=True)
            dest_zip.write(
                filename=self.dataset.name,
                arcname=os.path.basename(self.dataset.name),
                compress_type=zipfile.ZIP_DEFLATED,
            )
            dest_zip.close()

            # Store zip file in reprojected raster model
            self.rasterlayer.reprojected.rasterfile = File(
                open(dest_zip.filename, 'rb'),
                name=os.path.basename(dest_zip.filename)
            )
            self.rasterlayer.reprojected.save()

        self.log('Finished transforming raster.')

    def create_initial_histogram_buckets(self):
        """
        Gets the empty histogram arrays for statistics collection.
        """
        self.hist_values = []
        self.hist_bins = []

        for i, band in enumerate(self.dataset.bands):
            bandmeta = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer, band=i).first()
            self.hist_values.append(numpy.array(bandmeta.hist_values))
            self.hist_bins.append(numpy.array(bandmeta.hist_bins))

    def extract_metadata(self):
        """
        Extract and store metadata for the raster and its bands.
        """
        self.log('Extracting metadata from raster.')

        # Try to compute max zoom
        try:
            max_zoom = self.compute_max_zoom()
        except GDALException:
            raise RasterException('Failed to compute max zoom. Check the SRID of the raster.')

        # Extract global raster metadata
        meta = self.rasterlayer.metadata
        meta.uperleftx = self.dataset.origin.x
        meta.uperlefty = self.dataset.origin.y
        meta.width = self.dataset.width
        meta.height = self.dataset.height
        meta.scalex = self.dataset.scale.x
        meta.scaley = self.dataset.scale.y
        meta.skewx = self.dataset.skew.x
        meta.skewy = self.dataset.skew.y
        meta.numbands = len(self.dataset.bands)
        meta.srs_wkt = self.dataset.srs.wkt
        meta.srid = self.dataset.srs.srid
        meta.max_zoom = max_zoom
        meta.save()

        # Extract band metadata
        for i, band in enumerate(self.dataset.bands):
            bandmeta = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer, band=i).first()
            if not bandmeta:
                bandmeta = RasterLayerBandMetadata(rasterlayer=self.rasterlayer, band=i)

            bandmeta.nodata_value = band.nodata_value
            bandmeta.min = band.min
            bandmeta.max = band.max
            # Depending on Django version, the band statistics include std and mean.
            if hasattr(band, 'std'):
                bandmeta.std = band.std
            if hasattr(band, 'mean'):
                bandmeta.mean = band.mean
            bandmeta.save()

        self.log('Finished extracting metadata from raster.')

    def create_tiles(self, zoom_levels):
        """
        Create tiles for input zoom levels, either a list or an integer.
        """
        if isinstance(zoom_levels, int):
            self.populate_tile_level(zoom_levels)
        else:
            for zoom in zoom_levels:
                self.populate_tile_level(zoom)

    def populate_tile_level(self, zoom):
        """
        Create tiles for this raster at the given zoomlevel.

        This routine first snaps the raster to the grid of the zoomlevel,
        then creates  the tiles from the snapped raster.
        """
        # Abort if zoom level is above resolution of the raster layer
        if zoom > self.max_zoom:
            return
        elif zoom == self.max_zoom:
            self.create_initial_histogram_buckets()

        # Compute the tile x-y-z index range for the rasterlayer for this zoomlevel
        bbox = self.dataset.extent
        quadrants = utils.quadrants(bbox, zoom)

        self.log('Creating {0} tiles in {1} quadrants at zoom {2}.'.format(self.nr_of_tiles(zoom), len(quadrants), zoom))

        # Process quadrants in parallell
        for indexrange in quadrants:
            self.process_quadrant(indexrange, zoom)

        # Store histogram data
        if zoom == self.max_zoom:
            bandmetas = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer)
            for bandmeta in bandmetas:
                bandmeta.hist_values = self.hist_values[bandmeta.band].tolist()
                bandmeta.save()

        self.log('Finished parsing at zoom level {0}.'.format(zoom), zoom=zoom)

    _quadrant_count = 0

    def process_quadrant(self, indexrange, zoom):
        """
        Create raster tiles for a quadrant of tiles defined by a x-y-z index
        range and a zoom level.
        """
        # TODO Use a standalone celery task for this method in order to
        # gain speedup from parallelism.
        self._quadrant_count += 1
        self.log(
            'Starting tile creation for quadrant {0} at zoom level {1}'.format(self._quadrant_count, zoom),
            status=self.rasterlayer.parsestatus.CREATING_TILES
        )

        # Compute scale of tiles for this zoomlevel
        tilescale = utils.tile_scale(zoom)

        # Compute quadrant bounds and create destination file
        bounds = utils.tile_bounds(indexrange[0], indexrange[1], zoom)
        dest_file = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.tif')

        # Snap dataset to the quadrant
        snapped_dataset = self.dataset.warp({
            'name': dest_file.name,
            'origin': [bounds[0], bounds[3]],
            'scale': [tilescale, -tilescale],
            'width': (indexrange[2] - indexrange[0] + 1) * self.tilesize,
            'height': (indexrange[3] - indexrange[1] + 1) * self.tilesize,
        })

        # Create all tiles in this quadrant in batches
        batch = []
        for tilex in range(indexrange[0], indexrange[2] + 1):
            for tiley in range(indexrange[1], indexrange[3] + 1):
                # Calculate raster tile origin
                bounds = utils.tile_bounds(tilex, tiley, zoom)

                # Construct band data arrays
                pixeloffset = (
                    (tilex - indexrange[0]) * self.tilesize,
                    (tiley - indexrange[1]) * self.tilesize
                )

                band_data = [
                    {
                        'data': band.data(offset=pixeloffset, size=(self.tilesize, self.tilesize)),
                        'nodata_value': band.nodata_value
                    } for band in snapped_dataset.bands
                ]

                # Ignore tile if its only nodata.
                if all([numpy.all(dat['data'] == dat['nodata_value']) for dat in band_data]):
                    continue

                # Add tile data to histogram
                if zoom == self.max_zoom:
                    self.push_histogram(band_data)

                # Warp source raster into this tile (in memory)
                dest = GDALRaster({
                    'width': self.tilesize,
                    'height': self.tilesize,
                    'origin': [bounds[0], bounds[3]],
                    'scale': [tilescale, -tilescale],
                    'srid': WEB_MERCATOR_SRID,
                    'datatype': snapped_dataset.bands[0].datatype(),
                    'bands': band_data,
                })

                # Store tile in batch array
                batch.append(
                    RasterTile(
                        rast=dest,
                        rasterlayer_id=self.rasterlayer.id,
                        tilex=tilex,
                        tiley=tiley,
                        tilez=zoom
                    )
                )

                # Commit batch to database and reset it
                if len(batch) == self.batch_step_size:
                    RasterTile.objects.bulk_create(batch)
                    batch = []

        # Commit remaining objects
        if len(batch):
            RasterTile.objects.bulk_create(batch)

    def push_histogram(self, data):
        """
        Add data to band level histogram.
        """
        # Loop through bands of this tile
        for i, dat in enumerate(data):
            # Create histogram for new data with the same bins
            new_hist = numpy.histogram(dat['data'], bins=self.hist_bins[i])
            # Add counts of this tile to band metadata histogram
            self.hist_values[i] += new_hist[0]

    def drop_all_tiles(self):
        """
        Delete all existing tiles for this parser's rasterlayer.
        """
        self.log('Clearing all existing tiles.')
        self.rasterlayer.rastertile_set.all().delete()
        self.log('Finished clearing existing tiles.')

    def send_success_signal(self):
        """
        Send parser end signal for other dependencies to be handling new tiles.
        """
        self.log(
            'Successfully finished parsing raster',
            status=self.rasterlayer.parsestatus.FINISHED
        )
        rasterlayers_parser_ended.send(sender=self.rasterlayer.__class__, instance=self.rasterlayer)

    def compute_max_zoom(self):
        """
        Set max zoom property based on rasterlayer metadata.
        """
        # Return manual override value if provided
        if self.rasterlayer.max_zoom is not None:
            return self.rasterlayer.max_zoom

        if self.dataset.srs.srid == WEB_MERCATOR_SRID:
            # For rasters in web mercator, use the scale directly
            scale = abs(self.dataset.scale.x)
        else:
            # Create a line from the center of the raster to a point that is
            # one pixel width away from the center.
            xcenter = self.dataset.extent[0] + (self.dataset.extent[2] - self.dataset.extent[0]) / 2
            ycenter = self.dataset.extent[1] + (self.dataset.extent[3] - self.dataset.extent[1]) / 2
            linestring = 'LINESTRING({} {}, {} {})'.format(
                xcenter, ycenter, xcenter + self.dataset.scale.x, ycenter
            )
            line = OGRGeometry(linestring, srs=self.dataset.srs)

            # Tansform the line into web mercator.
            line.transform(WEB_MERCATOR_SRID)

            # Use the lenght of the transformed line as scale.
            scale = line.geos.length

        return utils.closest_zoomlevel(scale)

    @property
    def max_zoom(self):
        # Return manual override value if provided
        if self.rasterlayer.max_zoom is not None:
            return self.rasterlayer.max_zoom

        # Get max zoom from metadata
        if not hasattr(self.rasterlayer, 'metadata'):
            raise RasterException('Could not determine max zoom level.')
        max_zoom = self.rasterlayer.metadata.max_zoom

        # Reduce max zoom by one if zoomdown flag was disabled
        if not self.rasterlayer.next_higher:
            max_zoom -= 1

        return max_zoom

    def nr_of_tiles(self, zoom):
        """
        Compute the number of tiles for the rasterlayer on a given zoom level.
        """
        bbox = self.dataset.extent
        indexrange = utils.tile_index_range(bbox, zoom)
        return (indexrange[2] - indexrange[0] + 1) * (indexrange[3] - indexrange[1] + 1)
Exemple #46
0
class GDALRasterTests(SimpleTestCase):
    """
    Test a GDALRaster instance created from a file (GeoTiff).
    """
    def setUp(self):
        self.rs_path = os.path.join(os.path.dirname(__file__), '../data/rasters/raster.tif')
        self.rs = GDALRaster(self.rs_path)

    def test_rs_name_repr(self):
        self.assertEqual(self.rs_path, self.rs.name)
        self.assertRegex(repr(self.rs), r"<Raster object at 0x\w+>")

    def test_rs_driver(self):
        self.assertEqual(self.rs.driver.name, 'GTiff')

    def test_rs_size(self):
        self.assertEqual(self.rs.width, 163)
        self.assertEqual(self.rs.height, 174)

    def test_rs_srs(self):
        self.assertEqual(self.rs.srs.srid, 3086)
        self.assertEqual(self.rs.srs.units, (1.0, 'metre'))

    def test_rs_srid(self):
        rast = GDALRaster({
            'width': 16,
            'height': 16,
            'srid': 4326,
        })
        self.assertEqual(rast.srid, 4326)
        rast.srid = 3086
        self.assertEqual(rast.srid, 3086)

    def test_geotransform_and_friends(self):
        # Assert correct values for file based raster
        self.assertEqual(
            self.rs.geotransform,
            [511700.4680706557, 100.0, 0.0, 435103.3771231986, 0.0, -100.0]
        )
        self.assertEqual(self.rs.origin, [511700.4680706557, 435103.3771231986])
        self.assertEqual(self.rs.origin.x, 511700.4680706557)
        self.assertEqual(self.rs.origin.y, 435103.3771231986)
        self.assertEqual(self.rs.scale, [100.0, -100.0])
        self.assertEqual(self.rs.scale.x, 100.0)
        self.assertEqual(self.rs.scale.y, -100.0)
        self.assertEqual(self.rs.skew, [0, 0])
        self.assertEqual(self.rs.skew.x, 0)
        self.assertEqual(self.rs.skew.y, 0)
        # Create in-memory rasters and change gtvalues
        rsmem = GDALRaster(JSON_RASTER)
        # geotransform accepts both floats and ints
        rsmem.geotransform = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
        self.assertEqual(rsmem.geotransform, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0])
        rsmem.geotransform = range(6)
        self.assertEqual(rsmem.geotransform, [float(x) for x in range(6)])
        self.assertEqual(rsmem.origin, [0, 3])
        self.assertEqual(rsmem.origin.x, 0)
        self.assertEqual(rsmem.origin.y, 3)
        self.assertEqual(rsmem.scale, [1, 5])
        self.assertEqual(rsmem.scale.x, 1)
        self.assertEqual(rsmem.scale.y, 5)
        self.assertEqual(rsmem.skew, [2, 4])
        self.assertEqual(rsmem.skew.x, 2)
        self.assertEqual(rsmem.skew.y, 4)
        self.assertEqual(rsmem.width, 5)
        self.assertEqual(rsmem.height, 5)

    def test_geotransform_bad_inputs(self):
        rsmem = GDALRaster(JSON_RASTER)
        error_geotransforms = [
            [1, 2],
            [1, 2, 3, 4, 5, 'foo'],
            [1, 2, 3, 4, 5, 6, 'foo'],
        ]
        msg = 'Geotransform must consist of 6 numeric values.'
        for geotransform in error_geotransforms:
            with self.subTest(i=geotransform), self.assertRaisesMessage(ValueError, msg):
                rsmem.geotransform = geotransform

    def test_rs_extent(self):
        self.assertEqual(
            self.rs.extent,
            (511700.4680706557, 417703.3771231986, 528000.4680706557, 435103.3771231986)
        )

    def test_rs_bands(self):
        self.assertEqual(len(self.rs.bands), 1)
        self.assertIsInstance(self.rs.bands[0], GDALBand)

    def test_memory_based_raster_creation(self):
        # Create uint8 raster with full pixel data range (0-255)
        rast = GDALRaster({
            'datatype': 1,
            'width': 16,
            'height': 16,
            'srid': 4326,
            'bands': [{
                'data': range(256),
                'nodata_value': 255,
            }],
        })

        # Get array from raster
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # Assert data is same as original input
        self.assertEqual(result, list(range(256)))

    def test_file_based_raster_creation(self):
        # Prepare tempfile
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')

        # Create file-based raster from scratch
        GDALRaster({
            'datatype': self.rs.bands[0].datatype(),
            'driver': 'tif',
            'name': rstfile.name,
            'width': 163,
            'height': 174,
            'nr_of_bands': 1,
            'srid': self.rs.srs.wkt,
            'origin': (self.rs.origin.x, self.rs.origin.y),
            'scale': (self.rs.scale.x, self.rs.scale.y),
            'skew': (self.rs.skew.x, self.rs.skew.y),
            'bands': [{
                'data': self.rs.bands[0].data(),
                'nodata_value': self.rs.bands[0].nodata_value,
            }],
        })

        # Reload newly created raster from file
        restored_raster = GDALRaster(rstfile.name)
        self.assertEqual(restored_raster.srs.wkt, self.rs.srs.wkt)
        self.assertEqual(restored_raster.geotransform, self.rs.geotransform)
        if numpy:
            numpy.testing.assert_equal(
                restored_raster.bands[0].data(),
                self.rs.bands[0].data()
            )
        else:
            self.assertEqual(restored_raster.bands[0].data(), self.rs.bands[0].data())

    def test_vsi_raster_creation(self):
        # Open a raster as a file object.
        with open(self.rs_path, 'rb') as dat:
            # Instantiate a raster from the file binary buffer.
            vsimem = GDALRaster(dat.read())
        # The data of the in-memory file is equal to the source file.
        result = vsimem.bands[0].data()
        target = self.rs.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
            target = target.flatten().tolist()
        self.assertEqual(result, target)

    def test_vsi_raster_deletion(self):
        path = '/vsimem/raster.tif'
        # Create a vsi-based raster from scratch.
        vsimem = GDALRaster({
            'name': path,
            'driver': 'tif',
            'width': 4,
            'height': 4,
            'srid': 4326,
            'bands': [{
                'data': range(16),
            }],
        })
        # The virtual file exists.
        rst = GDALRaster(path)
        self.assertEqual(rst.width, 4)
        # Delete GDALRaster.
        del vsimem
        del rst
        # The virtual file has been removed.
        msg = 'Could not open the datasource at "/vsimem/raster.tif"'
        with self.assertRaisesMessage(GDALException, msg):
            GDALRaster(path)

    def test_vsi_invalid_buffer_error(self):
        msg = 'Failed creating VSI raster from the input buffer.'
        with self.assertRaisesMessage(GDALException, msg):
            GDALRaster(b'not-a-raster-buffer')

    def test_vsi_buffer_property(self):
        # Create a vsi-based raster from scratch.
        rast = GDALRaster({
            'name': '/vsimem/raster.tif',
            'driver': 'tif',
            'width': 4,
            'height': 4,
            'srid': 4326,
            'bands': [{
                'data': range(16),
            }],
        })
        # Do a round trip from raster to buffer to raster.
        result = GDALRaster(rast.vsi_buffer).bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to nodata value except on input block of ones.
        self.assertEqual(result, list(range(16)))
        # The vsi buffer is None for rasters that are not vsi based.
        self.assertIsNone(self.rs.vsi_buffer)

    def test_offset_size_and_shape_on_raster_creation(self):
        rast = GDALRaster({
            'datatype': 1,
            'width': 4,
            'height': 4,
            'srid': 4326,
            'bands': [{
                'data': (1,),
                'offset': (1, 1),
                'size': (2, 2),
                'shape': (1, 1),
                'nodata_value': 2,
            }],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to nodata value except on input block of ones.
        self.assertEqual(
            result,
            [2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2]
        )

    def test_set_nodata_value_on_raster_creation(self):
        # Create raster filled with nodata values.
        rast = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{'nodata_value': 23}],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # All band data is equal to nodata value.
        self.assertEqual(result, [23] * 4)

    def test_set_nodata_none_on_raster_creation(self):
        if GDAL_VERSION < (2, 1):
            self.skipTest("GDAL >= 2.1 is required for this test.")
        # Create raster without data and without nodata value.
        rast = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{'nodata_value': None}],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to zero because no nodata value has been specified.
        self.assertEqual(result, [0] * 4)

    def test_raster_metadata_property(self):
        data = self.rs.metadata
        self.assertEqual(data['DEFAULT'], {'AREA_OR_POINT': 'Area'})
        self.assertEqual(data['IMAGE_STRUCTURE'], {'INTERLEAVE': 'BAND'})

        # Create file-based raster from scratch
        source = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{'data': range(4), 'nodata_value': 99}],
        })
        # Set metadata on raster and on a band.
        metadata = {
            'DEFAULT': {'OWNER': 'Django', 'VERSION': '1.0', 'AREA_OR_POINT': 'Point'},
        }
        source.metadata = metadata
        source.bands[0].metadata = metadata
        self.assertEqual(source.metadata['DEFAULT'], metadata['DEFAULT'])
        self.assertEqual(source.bands[0].metadata['DEFAULT'], metadata['DEFAULT'])
        # Update metadata on raster.
        metadata = {
            'DEFAULT': {'VERSION': '2.0'},
        }
        source.metadata = metadata
        self.assertEqual(source.metadata['DEFAULT']['VERSION'], '2.0')
        # Remove metadata on raster.
        metadata = {
            'DEFAULT': {'OWNER': None},
        }
        source.metadata = metadata
        self.assertNotIn('OWNER', source.metadata['DEFAULT'])

    def test_raster_info_accessor(self):
        if GDAL_VERSION < (2, 1):
            msg = 'GDAL ≥ 2.1 is required for using the info property.'
            with self.assertRaisesMessage(ValueError, msg):
                self.rs.info
            return
        gdalinfo = """
        Driver: GTiff/GeoTIFF
        Files: {0}
        Size is 163, 174
        Coordinate System is:
        PROJCS["NAD83 / Florida GDL Albers",
            GEOGCS["NAD83",
                DATUM["North_American_Datum_1983",
                    SPHEROID["GRS 1980",6378137,298.257222101,
                        AUTHORITY["EPSG","7019"]],
                    TOWGS84[0,0,0,0,0,0,0],
                    AUTHORITY["EPSG","6269"]],
                PRIMEM["Greenwich",0,
                    AUTHORITY["EPSG","8901"]],
                UNIT["degree",0.0174532925199433,
                    AUTHORITY["EPSG","9122"]],
                AUTHORITY["EPSG","4269"]],
            PROJECTION["Albers_Conic_Equal_Area"],
            PARAMETER["standard_parallel_1",24],
            PARAMETER["standard_parallel_2",31.5],
            PARAMETER["latitude_of_center",24],
            PARAMETER["longitude_of_center",-84],
            PARAMETER["false_easting",400000],
            PARAMETER["false_northing",0],
            UNIT["metre",1,
                AUTHORITY["EPSG","9001"]],
            AXIS["X",EAST],
            AXIS["Y",NORTH],
            AUTHORITY["EPSG","3086"]]
        Origin = (511700.468070655711927,435103.377123198588379)
        Pixel Size = (100.000000000000000,-100.000000000000000)
        Metadata:
          AREA_OR_POINT=Area
        Image Structure Metadata:
          INTERLEAVE=BAND
        Corner Coordinates:
        Upper Left  (  511700.468,  435103.377) ( 82d51'46.16"W, 27d55' 1.53"N)
        Lower Left  (  511700.468,  417703.377) ( 82d51'52.04"W, 27d45'37.50"N)
        Upper Right (  528000.468,  435103.377) ( 82d41'48.81"W, 27d54'56.30"N)
        Lower Right (  528000.468,  417703.377) ( 82d41'55.54"W, 27d45'32.28"N)
        Center      (  519850.468,  426403.377) ( 82d46'50.64"W, 27d50'16.99"N)
        Band 1 Block=163x50 Type=Byte, ColorInterp=Gray
          NoData Value=15
        """.format(self.rs_path)
        # Data
        info_dyn = [line.strip() for line in self.rs.info.split('\n') if line.strip() != '']
        info_ref = [line.strip() for line in gdalinfo.split('\n') if line.strip() != '']
        self.assertEqual(info_dyn, info_ref)

    def test_compressed_file_based_raster_creation(self):
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
        # Make a compressed copy of an existing raster.
        compressed = self.rs.warp({'papsz_options': {'compress': 'packbits'}, 'name': rstfile.name})
        # Check physically if compression worked.
        self.assertLess(os.path.getsize(compressed.name), os.path.getsize(self.rs.name))
        # Create file-based raster with options from scratch.
        compressed = GDALRaster({
            'datatype': 1,
            'driver': 'tif',
            'name': rstfile.name,
            'width': 40,
            'height': 40,
            'srid': 3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(40 ^ 2),
                'nodata_value': 255,
            }],
            'papsz_options': {
                'compress': 'packbits',
                'pixeltype': 'signedbyte',
                'blockxsize': 23,
                'blockysize': 23,
            }
        })
        # Check if options used on creation are stored in metadata.
        # Reopening the raster ensures that all metadata has been written
        # to the file.
        compressed = GDALRaster(compressed.name)
        self.assertEqual(compressed.metadata['IMAGE_STRUCTURE']['COMPRESSION'], 'PACKBITS',)
        self.assertEqual(compressed.bands[0].metadata['IMAGE_STRUCTURE']['PIXELTYPE'], 'SIGNEDBYTE')
        if GDAL_VERSION >= (2, 1):
            self.assertIn('Block=40x23', compressed.info)

    def test_raster_warp(self):
        # Create in memory raster
        source = GDALRaster({
            'datatype': 1,
            'driver': 'MEM',
            'name': 'sourceraster',
            'width': 4,
            'height': 4,
            'nr_of_bands': 1,
            'srid': 3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(16),
                'nodata_value': 255,
            }],
        })

        # Test altering the scale, width, and height of a raster
        data = {
            'scale': [200, -200],
            'width': 2,
            'height': 2,
        }
        target = source.warp(data)
        self.assertEqual(target.width, data['width'])
        self.assertEqual(target.height, data['height'])
        self.assertEqual(target.scale, data['scale'])
        self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype())
        self.assertEqual(target.name, 'sourceraster_copy.MEM')
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(result, [5, 7, 13, 15])

        # Test altering the name and datatype (to float)
        data = {
            'name': '/path/to/targetraster.tif',
            'datatype': 6,
        }
        target = source.warp(data)
        self.assertEqual(target.bands[0].datatype(), 6)
        self.assertEqual(target.name, '/path/to/targetraster.tif')
        self.assertEqual(target.driver.name, 'MEM')
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(
            result,
            [0.0, 1.0, 2.0, 3.0,
             4.0, 5.0, 6.0, 7.0,
             8.0, 9.0, 10.0, 11.0,
             12.0, 13.0, 14.0, 15.0]
        )

    def test_raster_warp_nodata_zone(self):
        # Create in memory raster.
        source = GDALRaster({
            'datatype': 1,
            'driver': 'MEM',
            'width': 4,
            'height': 4,
            'srid': 3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(16),
                'nodata_value': 23,
            }],
        })
        # Warp raster onto a location that does not cover any pixels of the original.
        result = source.warp({'origin': (200000, 200000)}).bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # The result is an empty raster filled with the correct nodata value.
        self.assertEqual(result, [23] * 16)

    def test_raster_transform(self):
        # Prepare tempfile and nodata value
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
        ndv = 99

        # Create in file based raster
        source = GDALRaster({
            'datatype': 1,
            'driver': 'tif',
            'name': rstfile.name,
            'width': 5,
            'height': 5,
            'nr_of_bands': 1,
            'srid': 4326,
            'origin': (-5, 5),
            'scale': (2, -2),
            'skew': (0, 0),
            'bands': [{
                'data': range(25),
                'nodata_value': ndv,
            }],
        })

        # Transform raster into srid 4326.
        target = source.transform(3086)

        # Reload data from disk
        target = GDALRaster(target.name)

        self.assertEqual(target.srs.srid, 3086)
        self.assertEqual(target.width, 7)
        self.assertEqual(target.height, 7)
        self.assertEqual(target.bands[0].datatype(), source.bands[0].datatype())
        self.assertAlmostEqual(target.origin[0], 9124842.791079799, 3)
        self.assertAlmostEqual(target.origin[1], 1589911.6476407414, 3)
        self.assertAlmostEqual(target.scale[0], 223824.82664250192, 3)
        self.assertAlmostEqual(target.scale[1], -223824.82664250192, 3)
        self.assertEqual(target.skew, [0, 0])

        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # The reprojection of a raster that spans over a large area
        # skews the data matrix and might introduce nodata values.
        self.assertEqual(
            result,
            [
                ndv, ndv, ndv, ndv, 4, ndv, ndv,
                ndv, ndv, 2, 3, 9, ndv, ndv,
                ndv, 1, 2, 8, 13, 19, ndv,
                0, 6, 6, 12, 18, 18, 24,
                ndv, 10, 11, 16, 22, 23, ndv,
                ndv, ndv, 15, 21, 22, ndv, ndv,
                ndv, ndv, 20, ndv, ndv, ndv, ndv,
            ]
        )
class RasterLayerParser(object):
    """
    Class to parse raster layers.
    """
    def __init__(self, rasterlayer_id):
        self.rasterlayer = RasterLayer.objects.get(id=rasterlayer_id)

        # Set raster tilesize
        self.tilesize = int(getattr(settings, 'RASTER_TILESIZE', WEB_MERCATOR_TILESIZE))

    def log(self, msg, status=None, zoom=None):
        """
        Write a message to the parse log of the rasterlayer instance and update
        the parse status object.
        """
        parsestatus = self.rasterlayer.parsestatus
        parsestatus.refresh_from_db()

        if status is not None:
            parsestatus.status = status

        if zoom is not None and zoom not in parsestatus.tile_levels:
            parsestatus.tile_levels.append(zoom)
            parsestatus.tile_levels.sort()

        # Prepare datetime stamp for log
        now = '[{0}] '.format(datetime.datetime.now().strftime('%Y-%m-%d %T'))

        if parsestatus.log:
            now = '\n' + now

        parsestatus.log += now + msg
        parsestatus.save()

    def open_raster_file(self):
        """
        Get raster source file to extract tiles from.

        This makes a local copy of rasterfile, unzips the raster and reprojects
        it into web mercator if necessary. The reprojected raster is stored for
        reuse such that reprojection does only happen once.

        The local copy of the raster is needed if files are stored on remote
        storages.
        """
        reproj, created = RasterLayerReprojected.objects.get_or_create(rasterlayer=self.rasterlayer)
        # Check if the raster has already been reprojected
        has_reprojected = reproj.rasterfile.name not in (None, '')

        # Create workdir
        raster_workdir = getattr(settings, 'RASTER_WORKDIR', None)
        self.tmpdir = tempfile.mkdtemp(dir=raster_workdir)

        # Choose source for raster data, use the reprojected version if it exists.
        if self.rasterlayer.source_url and not has_reprojected:
            url_path = urlparse(self.rasterlayer.source_url).path
            filename = url_path.split('/')[-1]
            filepath = os.path.join(self.tmpdir, filename)
            urlretrieve(self.rasterlayer.source_url, filepath)
        else:
            if has_reprojected:
                rasterfile_source = reproj.rasterfile
            else:
                rasterfile_source = self.rasterlayer.rasterfile

            if not rasterfile_source.name:
                raise RasterException('No data source found. Provide a rasterfile or a source url.')

            # Copy raster file source to local folder
            filepath = os.path.join(self.tmpdir, os.path.basename(rasterfile_source.name))
            rasterfile = open(filepath, 'wb')
            for chunk in rasterfile_source.chunks():
                rasterfile.write(chunk)
            rasterfile.close()

        # If the raster file is compressed, decompress it, otherwise try to
        # open the source file directly.
        if os.path.splitext(filepath)[1].lower() == '.zip':
            # Open and extract zipfile
            zf = zipfile.ZipFile(filepath)
            zf.extractall(self.tmpdir)

            # Remove zipfile
            os.remove(filepath)

            # Get filelist from directory
            matches = []
            for root, dirnames, filenames in os.walk(self.tmpdir):
                for filename in fnmatch.filter(filenames, '*.*'):
                    matches.append(os.path.join(root, filename))

            # Open the first raster file found in the matched files.
            self.dataset = None
            for match in matches:
                try:
                    self.dataset = GDALRaster(match)
                    break
                except GDALException:
                    pass

            # Raise exception if no file could be opened by gdal.
            if not self.dataset:
                raise RasterException('Could not open rasterfile.')
        else:
            self.dataset = GDALRaster(filepath)

        # Override srid if provided
        if self.rasterlayer.srid:
            try:
                self.dataset = GDALRaster(self.dataset.name, write=True)
            except GDALException:
                raise RasterException(
                    'Could not override srid because the driver for this '
                    'type of raster does not support write mode.'
                )
            self.dataset.srs = self.rasterlayer.srid

    def reproject_rasterfile(self):
        """
        Reproject the rasterfile into web mercator.
        """
        # Return if reprojected rasterfile already exists.
        if hasattr(self.rasterlayer, 'reprojected') and self.rasterlayer.reprojected.rasterfile.name:
            return

        # Return if the raster already has the right projection
        # and nodata value is acceptable.
        if self.dataset.srs.srid == WEB_MERCATOR_SRID:
            # SRID was not manually specified.
            if self.rasterlayer.nodata in ('', None):
                return
            # All bands from dataset already have the same nodata value as the
            # one that was manually specified.
            if all([self.rasterlayer.nodata == band.nodata_value
                    for band in self.dataset.bands]):
                return
        else:
            # Log projection change if original raster is not in web mercator.
            self.log(
                'Transforming raster to SRID {0}'.format(WEB_MERCATOR_SRID),
                status=self.rasterlayer.parsestatus.REPROJECTING_RASTER,
            )

        # Reproject the dataset.
        self.dataset = self.dataset.transform(
            WEB_MERCATOR_SRID,
            driver=INTERMEDIATE_RASTER_FORMAT,
        )

        # Manually override nodata value if neccessary
        if self.rasterlayer.nodata not in ('', None):
            self.log(
                'Setting no data values to {0}.'.format(self.rasterlayer.nodata),
                status=self.rasterlayer.parsestatus.REPROJECTING_RASTER,
            )
            for band in self.dataset.bands:
                band.nodata_value = float(self.rasterlayer.nodata)

        # Compress reprojected raster file and store it
        if self.rasterlayer.store_reprojected:
            dest = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.zip')
            dest_zip = zipfile.ZipFile(dest.name, 'w', allowZip64=True)
            dest_zip.write(
                filename=self.dataset.name,
                arcname=os.path.basename(self.dataset.name),
                compress_type=zipfile.ZIP_DEFLATED,
            )
            dest_zip.close()

            # Store zip file in reprojected raster model
            self.rasterlayer.reprojected.rasterfile = File(open(dest_zip.filename, 'rb'))
            self.rasterlayer.reprojected.save()

        self.log('Finished transforming raster.')

    def create_initial_histogram_buckets(self):
        """
        Gets the empty histogram arrays for statistics collection.
        """
        self.hist_values = []
        self.hist_bins = []

        for i, band in enumerate(self.dataset.bands):
            bandmeta = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer, band=i).first()
            self.hist_values.append(numpy.array(bandmeta.hist_values))
            self.hist_bins.append(numpy.array(bandmeta.hist_bins))

    def extract_metadata(self):
        """
        Extract and store metadata for the raster and its bands.
        """
        self.log('Extracting metadata from raster.')

        # Try to compute max zoom
        try:
            max_zoom = self.compute_max_zoom()
        except GDALException:
            raise RasterException('Failed to compute max zoom. Check the SRID of the raster.')

        # Extract global raster metadata
        meta = self.rasterlayer.metadata
        meta.uperleftx = self.dataset.origin.x
        meta.uperlefty = self.dataset.origin.y
        meta.width = self.dataset.width
        meta.height = self.dataset.height
        meta.scalex = self.dataset.scale.x
        meta.scaley = self.dataset.scale.y
        meta.skewx = self.dataset.skew.x
        meta.skewy = self.dataset.skew.y
        meta.numbands = len(self.dataset.bands)
        meta.srs_wkt = self.dataset.srs.wkt
        meta.srid = self.dataset.srs.srid
        meta.max_zoom = max_zoom
        meta.save()

        # Extract band metadata
        for i, band in enumerate(self.dataset.bands):
            bandmeta = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer, band=i).first()
            if not bandmeta:
                bandmeta = RasterLayerBandMetadata(rasterlayer=self.rasterlayer, band=i)

            bandmeta.nodata_value = band.nodata_value
            bandmeta.min = band.min
            bandmeta.max = band.max
            # Depending on Django version, the band statistics include std and mean.
            if hasattr(band, 'std'):
                bandmeta.std = band.std
            if hasattr(band, 'mean'):
                bandmeta.mean = band.mean
            bandmeta.save()

        self.log('Finished extracting metadata from raster.')

    def create_tiles(self, zoom_levels):
        """
        Create tiles for input zoom levels, either a list or an integer.
        """
        if isinstance(zoom_levels, int):
            self.populate_tile_level(zoom_levels)
        else:
            for zoom in zoom_levels:
                self.populate_tile_level(zoom)

    def populate_tile_level(self, zoom):
        """
        Create tiles for this raster at the given zoomlevel.

        This routine first snaps the raster to the grid of the zoomlevel,
        then creates  the tiles from the snapped raster.
        """
        # Abort if zoom level is above resolution of the raster layer
        if zoom > self.max_zoom:
            return
        elif zoom == self.max_zoom:
            self.create_initial_histogram_buckets()

        # Compute the tile x-y-z index range for the rasterlayer for this zoomlevel
        bbox = self.dataset.extent
        quadrants = utils.quadrants(bbox, zoom)

        self.log('Creating {0} tiles in {1} quadrants at zoom {2}.'.format(self.nr_of_tiles(zoom), len(quadrants), zoom))

        # Process quadrants in parallell
        quadrant_task_group = group(self.process_quadrant.si(indexrange, zoom) for indexrange in quadrants)
        quadrant_task_group.apply()

        # Store histogram data
        if zoom == self.max_zoom:
            bandmetas = RasterLayerBandMetadata.objects.filter(rasterlayer=self.rasterlayer)
            for bandmeta in bandmetas:
                bandmeta.hist_values = self.hist_values[bandmeta.band].tolist()
                bandmeta.save()

        self.log('Finished parsing at zoom level {0}.'.format(zoom), zoom=zoom)

    _quadrant_count = 0

    @current_app.task(filter=task_method)
    def process_quadrant(self, indexrange, zoom):
        """
        Create raster tiles for a quadrant of tiles defined by a x-y-z index
        range and a zoom level.
        """
        self._quadrant_count += 1
        self.log(
            'Starting tile creation for quadrant {0} at zoom level {1}'.format(self._quadrant_count, zoom),
            status=self.rasterlayer.parsestatus.CREATING_TILES
        )

        # Compute scale of tiles for this zoomlevel
        tilescale = utils.tile_scale(zoom)

        # Compute quadrant bounds and create destination file
        bounds = utils.tile_bounds(indexrange[0], indexrange[1], zoom)
        dest_file = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix='.tif')

        # Snap dataset to the quadrant
        snapped_dataset = self.dataset.warp({
            'name': dest_file.name,
            'origin': [bounds[0], bounds[3]],
            'scale': [tilescale, -tilescale],
            'width': (indexrange[2] - indexrange[0] + 1) * self.tilesize,
            'height': (indexrange[3] - indexrange[1] + 1) * self.tilesize,
        })

        # Create all tiles in this quadrant in batches
        batch = []
        for tilex in range(indexrange[0], indexrange[2] + 1):
            for tiley in range(indexrange[1], indexrange[3] + 1):
                # Calculate raster tile origin
                bounds = utils.tile_bounds(tilex, tiley, zoom)

                # Construct band data arrays
                pixeloffset = (
                    (tilex - indexrange[0]) * self.tilesize,
                    (tiley - indexrange[1]) * self.tilesize
                )

                band_data = [
                    {
                        'data': band.data(offset=pixeloffset, size=(self.tilesize, self.tilesize)),
                        'nodata_value': band.nodata_value
                    } for band in snapped_dataset.bands
                ]

                # Ignore tile if its only nodata.
                if all([numpy.all(dat['data'] == dat['nodata_value']) for dat in band_data]):
                    continue

                # Add tile data to histogram
                if zoom == self.max_zoom:
                    self.push_histogram(band_data)

                # Warp source raster into this tile (in memory)
                dest = GDALRaster({
                    'width': self.tilesize,
                    'height': self.tilesize,
                    'origin': [bounds[0], bounds[3]],
                    'scale': [tilescale, -tilescale],
                    'srid': WEB_MERCATOR_SRID,
                    'datatype': snapped_dataset.bands[0].datatype(),
                    'bands': band_data,
                })

                # Store tile in batch array
                batch.append(
                    RasterTile(
                        rast=dest,
                        rasterlayer_id=self.rasterlayer.id,
                        tilex=tilex,
                        tiley=tiley,
                        tilez=zoom
                    )
                )

                # Commit batch to database and reset it
                if len(batch) == BATCH_STEP_SIZE:
                    RasterTile.objects.bulk_create(batch)
                    batch = []

        # Commit remaining objects
        if len(batch):
            RasterTile.objects.bulk_create(batch)

    def push_histogram(self, data):
        """
        Add data to band level histogram.
        """
        # Loop through bands of this tile
        for i, dat in enumerate(data):
            # Create histogram for new data with the same bins
            new_hist = numpy.histogram(dat['data'], bins=self.hist_bins[i])
            # Add counts of this tile to band metadata histogram
            self.hist_values[i] += new_hist[0]

    def drop_all_tiles(self):
        """
        Delete all existing tiles for this parser's rasterlayer.
        """
        self.log('Clearing all existing tiles.')
        self.rasterlayer.rastertile_set.all().delete()
        self.log('Finished clearing existing tiles.')

    def send_success_signal(self):
        """
        Send parser end signal for other dependencies to be handling new tiles.
        """
        self.log(
            'Successfully finished parsing raster',
            status=self.rasterlayer.parsestatus.FINISHED
        )
        rasterlayers_parser_ended.send(sender=self.rasterlayer.__class__, instance=self.rasterlayer)

    def compute_max_zoom(self):
        """
        Set max zoom property based on rasterlayer metadata.
        """
        # Return manual override value if provided
        if self.rasterlayer.max_zoom is not None:
            return self.rasterlayer.max_zoom

        if self.dataset.srs.srid == WEB_MERCATOR_SRID:
            # For rasters in web mercator, use the scale directly
            scale = abs(self.dataset.scale.x)
        else:
            # Create a line from the center of the raster to a point that is
            # one pixel width away from the center.
            xcenter = self.dataset.extent[0] + (self.dataset.extent[2] - self.dataset.extent[0]) / 2
            ycenter = self.dataset.extent[1] + (self.dataset.extent[3] - self.dataset.extent[1]) / 2
            linestring = 'LINESTRING({} {}, {} {})'.format(
                xcenter, ycenter, xcenter + self.dataset.scale.x, ycenter
            )
            line = OGRGeometry(linestring, srs=self.dataset.srs)

            # Tansform the line into web mercator.
            line.transform(WEB_MERCATOR_SRID)

            # Use the lenght of the transformed line as scale.
            scale = line.geos.length

        return utils.closest_zoomlevel(scale)

    @property
    def max_zoom(self):
        # Return manual override value if provided
        if self.rasterlayer.max_zoom is not None:
            return self.rasterlayer.max_zoom

        # Get max zoom from metadata
        if not hasattr(self.rasterlayer, 'metadata'):
            raise RasterException('Could not determine max zoom level.')
        max_zoom = self.rasterlayer.metadata.max_zoom

        # Reduce max zoom by one if zoomdown flag was disabled
        if not self.rasterlayer.next_higher:
            max_zoom -= 1

        return max_zoom

    def nr_of_tiles(self, zoom):
        """
        Compute the number of tiles for the rasterlayer on a given zoom level.
        """
        bbox = self.dataset.extent
        indexrange = utils.tile_index_range(bbox, zoom)
        return (indexrange[2] - indexrange[0] + 1) * (indexrange[3] - indexrange[1] + 1)
Exemple #48
0
    def process(self):
        """
        This method contains the logic for processing tasks asynchronously
        from a background thread or from the scheduler. Here tasks that are
        ready to be processed execute some logic. This could be communication
        with a processing node or executing a pending action.
        """

        try:
            if self.processing_node:
                # Need to process some images (UUID not yet set and task doesn't have pending actions)?
                if not self.uuid and self.pending_action is None:
                    logger.info("Processing... {}".format(self))

                    images = [
                        image.path() for image in self.imageupload_set.all()
                    ]

                    try:
                        # This takes a while
                        uuid = self.processing_node.process_new_task(
                            images, self.name, self.options)

                        # Refresh task object before committing change
                        self.refresh_from_db()
                        self.uuid = uuid
                        self.save()

                        # TODO: log process has started processing

                    except ProcessingError as e:
                        self.set_failure(str(e))

            if self.pending_action is not None:
                try:
                    if self.pending_action == pending_actions.CANCEL:
                        # Do we need to cancel the task on the processing node?
                        logger.info("Canceling {}".format(self))
                        if self.processing_node and self.uuid:
                            self.processing_node.cancel_task(self.uuid)
                            self.pending_action = None
                            self.status = None
                            self.save()
                        else:
                            raise ProcessingError(
                                "Cannot cancel a task that has no processing node or UUID"
                            )

                    elif self.pending_action == pending_actions.RESTART:
                        logger.info("Restarting {}".format(self))
                        if self.processing_node:

                            # Check if the UUID is still valid, as processing nodes purge
                            # results after a set amount of time, the UUID might have eliminated.
                            uuid_still_exists = False

                            if self.uuid:
                                try:
                                    info = self.processing_node.get_task_info(
                                        self.uuid)
                                    uuid_still_exists = info[
                                        'uuid'] == self.uuid
                                except ProcessingError:
                                    pass

                            if uuid_still_exists:
                                # Good to go
                                self.processing_node.restart_task(self.uuid)
                            else:
                                # Task has been purged (or processing node is offline)
                                # Process this as a new task
                                # Removing its UUID will cause the scheduler
                                # to process this the next tick
                                self.uuid = ''

                            self.console_output = ""
                            self.processing_time = -1
                            self.status = None
                            self.last_error = None
                            self.pending_action = None
                            self.save()
                        else:
                            raise ProcessingError(
                                "Cannot restart a task that has no processing node or UUID"
                            )

                    elif self.pending_action == pending_actions.REMOVE:
                        logger.info("Removing {}".format(self))
                        if self.processing_node and self.uuid:
                            # Attempt to delete the resources on the processing node
                            # We don't care if this fails, as resources on processing nodes
                            # Are expected to be purged on their own after a set amount of time anyway
                            try:
                                self.processing_node.remove_task(self.uuid)
                            except ProcessingException:
                                pass

                        # What's more important is that we delete our task properly here
                        self.delete()

                        # Stop right here!
                        return

                except ProcessingError as e:
                    self.last_error = str(e)
                    self.save()

            if self.processing_node:
                # Need to update status (first time, queued or running?)
                if self.uuid and self.status in [
                        None, status_codes.QUEUED, status_codes.RUNNING
                ]:
                    # Update task info from processing node
                    try:
                        info = self.processing_node.get_task_info(self.uuid)

                        self.processing_time = info["processingTime"]
                        self.status = info["status"]["code"]

                        current_lines_count = len(
                            self.console_output.split("\n")) - 1
                        self.console_output += self.processing_node.get_task_console_output(
                            self.uuid, current_lines_count)

                        if "errorMessage" in info["status"]:
                            self.last_error = info["status"]["errorMessage"]

                        # Has the task just been canceled, failed, or completed?
                        if self.status in [
                                status_codes.FAILED, status_codes.COMPLETED,
                                status_codes.CANCELED
                        ]:
                            logger.info("Processing status: {} for {}".format(
                                self.status, self))

                            if self.status == status_codes.COMPLETED:
                                try:
                                    assets_dir = self.assets_path("")
                                    if not os.path.exists(assets_dir):
                                        os.makedirs(assets_dir)

                                    logger.info(
                                        "Downloading all.zip for {}".format(
                                            self))

                                    # Download all assets
                                    zip_stream = self.processing_node.download_task_asset(
                                        self.uuid, "all.zip")
                                    zip_path = os.path.join(
                                        assets_dir, "all.zip")
                                    with open(zip_path, 'wb') as fd:
                                        for chunk in zip_stream.iter_content(
                                                4096):
                                            fd.write(chunk)

                                    logger.info(
                                        "Done downloading all.zip for {}".
                                        format(self))

                                    # Extract from zip
                                    with zipfile.ZipFile(zip_path,
                                                         "r") as zip_h:
                                        zip_h.extractall(assets_dir)

                                    logger.info(
                                        "Extracted all.zip for {}".format(
                                            self))

                                    # Add to database orthophoto
                                    orthophoto_path = os.path.realpath(
                                        self.assets_path(
                                            "odm_orthophoto",
                                            "odm_orthophoto.tif"))
                                    if os.path.exists(orthophoto_path):
                                        orthophoto = GDALRaster(
                                            orthophoto_path, write=True)

                                        # We need to transform to 4326 before we can store it
                                        # as an offdb raster field
                                        orthophoto_4326_path = os.path.realpath(
                                            self.assets_path(
                                                "odm_orthophoto",
                                                "odm_orthophoto_4326.tif"))
                                        self.orthophoto = orthophoto.transform(
                                            4326, 'GTiff',
                                            orthophoto_4326_path)

                                        logger.info(
                                            "Imported orthophoto {} for {}".
                                            format(orthophoto_4326_path, self))

                                    self.save()
                                except ProcessingError as e:
                                    self.set_failure(str(e))
                            else:
                                # FAILED, CANCELED
                                self.save()
                        else:
                            # Still waiting...
                            self.save()
                    except ProcessingError as e:
                        self.set_failure(str(e))
        except (ConnectionRefusedError, ConnectionError) as e:
            logger.warning(
                "{} cannot communicate with processing node: {}".format(
                    self, str(e)))
        except ProcessingTimeout as e:
            logger.warning(
                "{} timed out with error: {}. We'll try reprocessing at the next tick."
                .format(self, str(e)))
Exemple #49
0
    def test_all_gis_lookups_with_rasters(self):
        """
        Evaluate all possible lookups for all input combinations (i.e.
        raster-raster, raster-geom, geom-raster) and for projected and
        unprojected coordinate systems. This test just checks that the lookup
        can be called, but doesn't check if the result makes logical sense.
        """
        from django.contrib.gis.db.backends.postgis.operations import PostGISOperations

        # Create test raster and geom.
        rast = GDALRaster(json.loads(JSON_RASTER))
        stx_pnt = GEOSGeometry(
            'POINT (-95.370401017314293 29.704867409475465)', 4326)
        stx_pnt.transform(3086)

        # Loop through all the GIS lookups.
        for name, lookup in BaseSpatialField.get_lookups().items():
            if not isinstance(lookup, GISLookup):
                continue
            # Construct lookup filter strings.
            combo_keys = [
                field + name for field in [
                    'rast__',
                    'rast__',
                    'rastprojected__0__',
                    'rast__',
                    'rastprojected__',
                    'geom__',
                    'rast__',
                ]
            ]
            if issubclass(lookup, DistanceLookupBase):
                # Set lookup values for distance lookups.
                combo_values = [
                    (rast, 50, 'spheroid'),
                    (rast, 0, 50, 'spheroid'),
                    (rast, 0, D(km=1)),
                    (stx_pnt, 0, 500),
                    (stx_pnt, D(km=1000)),
                    (rast, 500),
                    (json.loads(JSON_RASTER), 500),
                ]
            elif name == 'relate':
                # Set lookup values for the relate lookup.
                combo_values = [
                    (rast, 'T*T***FF*'),
                    (rast, 0, 'T*T***FF*'),
                    (rast, 0, 'T*T***FF*'),
                    (stx_pnt, 0, 'T*T***FF*'),
                    (stx_pnt, 'T*T***FF*'),
                    (rast, 'T*T***FF*'),
                    (json.loads(JSON_RASTER), 'T*T***FF*'),
                ]
            elif name == 'isvalid':
                # The isvalid lookup doesn't make sense for rasters.
                continue
            elif PostGISOperations.gis_operators[name].func:
                # Set lookup values for all function based operators.
                combo_values = [
                    rast, (rast, 0), (rast, 0), (stx_pnt, 0), stx_pnt, rast,
                    rast,
                    json.loads(JSON_RASTER)
                ]
            else:
                # Override band lookup for these, as it's not supported.
                combo_keys[2] = 'rastprojected__' + name
                # Set lookup values for all other operators.
                combo_values = [
                    rast, rast, rast, stx_pnt, stx_pnt, rast, rast,
                    json.loads(JSON_RASTER)
                ]

            # Create query filter combinations.
            combos = [{x[0]: x[1]} for x in zip(combo_keys, combo_values)]

            for combo in combos:
                # Apply this query filter.
                qs = RasterModel.objects.filter(**combo)

                # Evaluate normal filter qs.
                self.assertIn(qs.count(), [0, 1])

            # Evaluate on conditional Q expressions.
            qs = RasterModel.objects.filter(Q(**combos[0]) & Q(**combos[1]))
            self.assertIn(qs.count(), [0, 1])
Exemple #50
0
class GDALRasterTests(SimpleTestCase):
    """
    Test a GDALRaster instance created from a file (GeoTiff).
    """
    def setUp(self):
        self.rs_path = os.path.join(os.path.dirname(__file__),
                                    '../data/rasters/raster.tif')
        self.rs = GDALRaster(self.rs_path)

    def test_rs_name_repr(self):
        self.assertEqual(self.rs_path, self.rs.name)
        self.assertRegex(repr(self.rs), r"<Raster object at 0x\w+>")

    def test_rs_driver(self):
        self.assertEqual(self.rs.driver.name, 'GTiff')

    def test_rs_size(self):
        self.assertEqual(self.rs.width, 163)
        self.assertEqual(self.rs.height, 174)

    def test_rs_srs(self):
        self.assertEqual(self.rs.srs.srid, 3086)
        self.assertEqual(self.rs.srs.units, (1.0, 'metre'))

    def test_rs_srid(self):
        rast = GDALRaster({
            'width': 16,
            'height': 16,
            'srid': 4326,
        })
        self.assertEqual(rast.srid, 4326)
        rast.srid = 3086
        self.assertEqual(rast.srid, 3086)

    def test_geotransform_and_friends(self):
        # Assert correct values for file based raster
        self.assertEqual(
            self.rs.geotransform,
            [511700.4680706557, 100.0, 0.0, 435103.3771231986, 0.0, -100.0])
        self.assertEqual(self.rs.origin,
                         [511700.4680706557, 435103.3771231986])
        self.assertEqual(self.rs.origin.x, 511700.4680706557)
        self.assertEqual(self.rs.origin.y, 435103.3771231986)
        self.assertEqual(self.rs.scale, [100.0, -100.0])
        self.assertEqual(self.rs.scale.x, 100.0)
        self.assertEqual(self.rs.scale.y, -100.0)
        self.assertEqual(self.rs.skew, [0, 0])
        self.assertEqual(self.rs.skew.x, 0)
        self.assertEqual(self.rs.skew.y, 0)
        # Create in-memory rasters and change gtvalues
        rsmem = GDALRaster(JSON_RASTER)
        # geotransform accepts both floats and ints
        rsmem.geotransform = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
        self.assertEqual(rsmem.geotransform, [0.0, 1.0, 2.0, 3.0, 4.0, 5.0])
        rsmem.geotransform = range(6)
        self.assertEqual(rsmem.geotransform, [float(x) for x in range(6)])
        self.assertEqual(rsmem.origin, [0, 3])
        self.assertEqual(rsmem.origin.x, 0)
        self.assertEqual(rsmem.origin.y, 3)
        self.assertEqual(rsmem.scale, [1, 5])
        self.assertEqual(rsmem.scale.x, 1)
        self.assertEqual(rsmem.scale.y, 5)
        self.assertEqual(rsmem.skew, [2, 4])
        self.assertEqual(rsmem.skew.x, 2)
        self.assertEqual(rsmem.skew.y, 4)
        self.assertEqual(rsmem.width, 5)
        self.assertEqual(rsmem.height, 5)

    def test_geotransform_bad_inputs(self):
        rsmem = GDALRaster(JSON_RASTER)
        error_geotransforms = [
            [1, 2],
            [1, 2, 3, 4, 5, 'foo'],
            [1, 2, 3, 4, 5, 6, 'foo'],
        ]
        msg = 'Geotransform must consist of 6 numeric values.'
        for geotransform in error_geotransforms:
            with self.subTest(i=geotransform), self.assertRaisesMessage(
                    ValueError, msg):
                rsmem.geotransform = geotransform

    def test_rs_extent(self):
        self.assertEqual(self.rs.extent,
                         (511700.4680706557, 417703.3771231986,
                          528000.4680706557, 435103.3771231986))

    def test_rs_bands(self):
        self.assertEqual(len(self.rs.bands), 1)
        self.assertIsInstance(self.rs.bands[0], GDALBand)

    def test_memory_based_raster_creation(self):
        # Create uint8 raster with full pixel data range (0-255)
        rast = GDALRaster({
            'datatype':
            1,
            'width':
            16,
            'height':
            16,
            'srid':
            4326,
            'bands': [{
                'data': range(256),
                'nodata_value': 255,
            }],
        })

        # Get array from raster
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # Assert data is same as original input
        self.assertEqual(result, list(range(256)))

    def test_file_based_raster_creation(self):
        # Prepare tempfile
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')

        # Create file-based raster from scratch
        GDALRaster({
            'datatype':
            self.rs.bands[0].datatype(),
            'driver':
            'tif',
            'name':
            rstfile.name,
            'width':
            163,
            'height':
            174,
            'nr_of_bands':
            1,
            'srid':
            self.rs.srs.wkt,
            'origin': (self.rs.origin.x, self.rs.origin.y),
            'scale': (self.rs.scale.x, self.rs.scale.y),
            'skew': (self.rs.skew.x, self.rs.skew.y),
            'bands': [{
                'data': self.rs.bands[0].data(),
                'nodata_value': self.rs.bands[0].nodata_value,
            }],
        })

        # Reload newly created raster from file
        restored_raster = GDALRaster(rstfile.name)
        self.assertEqual(restored_raster.srs.wkt, self.rs.srs.wkt)
        self.assertEqual(restored_raster.geotransform, self.rs.geotransform)
        if numpy:
            numpy.testing.assert_equal(restored_raster.bands[0].data(),
                                       self.rs.bands[0].data())
        else:
            self.assertEqual(restored_raster.bands[0].data(),
                             self.rs.bands[0].data())

    def test_vsi_raster_creation(self):
        # Open a raster as a file object.
        with open(self.rs_path, 'rb') as dat:
            # Instantiate a raster from the file binary buffer.
            vsimem = GDALRaster(dat.read())
        # The data of the in-memory file is equal to the source file.
        result = vsimem.bands[0].data()
        target = self.rs.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
            target = target.flatten().tolist()
        self.assertEqual(result, target)

    def test_vsi_raster_deletion(self):
        path = '/vsimem/raster.tif'
        # Create a vsi-based raster from scratch.
        vsimem = GDALRaster({
            'name': path,
            'driver': 'tif',
            'width': 4,
            'height': 4,
            'srid': 4326,
            'bands': [{
                'data': range(16),
            }],
        })
        # The virtual file exists.
        rst = GDALRaster(path)
        self.assertEqual(rst.width, 4)
        # Delete GDALRaster.
        del vsimem
        del rst
        # The virtual file has been removed.
        msg = 'Could not open the datasource at "/vsimem/raster.tif"'
        with self.assertRaisesMessage(GDALException, msg):
            GDALRaster(path)

    def test_vsi_invalid_buffer_error(self):
        msg = 'Failed creating VSI raster from the input buffer.'
        with self.assertRaisesMessage(GDALException, msg):
            GDALRaster(b'not-a-raster-buffer')

    def test_vsi_buffer_property(self):
        # Create a vsi-based raster from scratch.
        rast = GDALRaster({
            'name': '/vsimem/raster.tif',
            'driver': 'tif',
            'width': 4,
            'height': 4,
            'srid': 4326,
            'bands': [{
                'data': range(16),
            }],
        })
        # Do a round trip from raster to buffer to raster.
        result = GDALRaster(rast.vsi_buffer).bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to nodata value except on input block of ones.
        self.assertEqual(result, list(range(16)))
        # The vsi buffer is None for rasters that are not vsi based.
        self.assertIsNone(self.rs.vsi_buffer)

    def test_offset_size_and_shape_on_raster_creation(self):
        rast = GDALRaster({
            'datatype':
            1,
            'width':
            4,
            'height':
            4,
            'srid':
            4326,
            'bands': [{
                'data': (1, ),
                'offset': (1, 1),
                'size': (2, 2),
                'shape': (1, 1),
                'nodata_value': 2,
            }],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to nodata value except on input block of ones.
        self.assertEqual(result,
                         [2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2])

    def test_set_nodata_value_on_raster_creation(self):
        # Create raster filled with nodata values.
        rast = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{
                'nodata_value': 23
            }],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # All band data is equal to nodata value.
        self.assertEqual(result, [23] * 4)

    def test_set_nodata_none_on_raster_creation(self):
        if GDAL_VERSION < (2, 1):
            self.skipTest("GDAL >= 2.1 is required for this test.")
        # Create raster without data and without nodata value.
        rast = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{
                'nodata_value': None
            }],
        })
        # Get array from raster.
        result = rast.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # Band data is equal to zero because no nodata value has been specified.
        self.assertEqual(result, [0] * 4)

    def test_raster_metadata_property(self):
        data = self.rs.metadata
        self.assertEqual(data['DEFAULT'], {'AREA_OR_POINT': 'Area'})
        self.assertEqual(data['IMAGE_STRUCTURE'], {'INTERLEAVE': 'BAND'})

        # Create file-based raster from scratch
        source = GDALRaster({
            'datatype': 1,
            'width': 2,
            'height': 2,
            'srid': 4326,
            'bands': [{
                'data': range(4),
                'nodata_value': 99
            }],
        })
        # Set metadata on raster and on a band.
        metadata = {
            'DEFAULT': {
                'OWNER': 'Django',
                'VERSION': '1.0',
                'AREA_OR_POINT': 'Point'
            },
        }
        source.metadata = metadata
        source.bands[0].metadata = metadata
        self.assertEqual(source.metadata['DEFAULT'], metadata['DEFAULT'])
        self.assertEqual(source.bands[0].metadata['DEFAULT'],
                         metadata['DEFAULT'])
        # Update metadata on raster.
        metadata = {
            'DEFAULT': {
                'VERSION': '2.0'
            },
        }
        source.metadata = metadata
        self.assertEqual(source.metadata['DEFAULT']['VERSION'], '2.0')
        # Remove metadata on raster.
        metadata = {
            'DEFAULT': {
                'OWNER': None
            },
        }
        source.metadata = metadata
        self.assertNotIn('OWNER', source.metadata['DEFAULT'])

    def test_raster_info_accessor(self):
        if GDAL_VERSION < (2, 1):
            msg = 'GDAL ≥ 2.1 is required for using the info property.'
            with self.assertRaisesMessage(ValueError, msg):
                self.rs.info
            return
        gdalinfo = """
        Driver: GTiff/GeoTIFF
        Files: {0}
        Size is 163, 174
        Coordinate System is:
        PROJCS["NAD83 / Florida GDL Albers",
            GEOGCS["NAD83",
                DATUM["North_American_Datum_1983",
                    SPHEROID["GRS 1980",6378137,298.257222101,
                        AUTHORITY["EPSG","7019"]],
                    TOWGS84[0,0,0,0,0,0,0],
                    AUTHORITY["EPSG","6269"]],
                PRIMEM["Greenwich",0,
                    AUTHORITY["EPSG","8901"]],
                UNIT["degree",0.0174532925199433,
                    AUTHORITY["EPSG","9122"]],
                AUTHORITY["EPSG","4269"]],
            PROJECTION["Albers_Conic_Equal_Area"],
            PARAMETER["standard_parallel_1",24],
            PARAMETER["standard_parallel_2",31.5],
            PARAMETER["latitude_of_center",24],
            PARAMETER["longitude_of_center",-84],
            PARAMETER["false_easting",400000],
            PARAMETER["false_northing",0],
            UNIT["metre",1,
                AUTHORITY["EPSG","9001"]],
            AXIS["X",EAST],
            AXIS["Y",NORTH],
            AUTHORITY["EPSG","3086"]]
        Origin = (511700.468070655711927,435103.377123198588379)
        Pixel Size = (100.000000000000000,-100.000000000000000)
        Metadata:
          AREA_OR_POINT=Area
        Image Structure Metadata:
          INTERLEAVE=BAND
        Corner Coordinates:
        Upper Left  (  511700.468,  435103.377) ( 82d51'46.16"W, 27d55' 1.53"N)
        Lower Left  (  511700.468,  417703.377) ( 82d51'52.04"W, 27d45'37.50"N)
        Upper Right (  528000.468,  435103.377) ( 82d41'48.81"W, 27d54'56.30"N)
        Lower Right (  528000.468,  417703.377) ( 82d41'55.54"W, 27d45'32.28"N)
        Center      (  519850.468,  426403.377) ( 82d46'50.64"W, 27d50'16.99"N)
        Band 1 Block=163x50 Type=Byte, ColorInterp=Gray
          NoData Value=15
        """.format(self.rs_path)
        # Data
        info_dyn = [
            line.strip() for line in self.rs.info.split('\n')
            if line.strip() != ''
        ]
        info_ref = [
            line.strip() for line in gdalinfo.split('\n') if line.strip() != ''
        ]
        self.assertEqual(info_dyn, info_ref)

    def test_compressed_file_based_raster_creation(self):
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
        # Make a compressed copy of an existing raster.
        compressed = self.rs.warp({
            'papsz_options': {
                'compress': 'packbits'
            },
            'name': rstfile.name
        })
        # Check physically if compression worked.
        self.assertLess(os.path.getsize(compressed.name),
                        os.path.getsize(self.rs.name))
        # Create file-based raster with options from scratch.
        compressed = GDALRaster({
            'datatype':
            1,
            'driver':
            'tif',
            'name':
            rstfile.name,
            'width':
            40,
            'height':
            40,
            'srid':
            3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(40 ^ 2),
                'nodata_value': 255,
            }],
            'papsz_options': {
                'compress': 'packbits',
                'pixeltype': 'signedbyte',
                'blockxsize': 23,
                'blockysize': 23,
            }
        })
        # Check if options used on creation are stored in metadata.
        # Reopening the raster ensures that all metadata has been written
        # to the file.
        compressed = GDALRaster(compressed.name)
        self.assertEqual(
            compressed.metadata['IMAGE_STRUCTURE']['COMPRESSION'],
            'PACKBITS',
        )
        self.assertEqual(
            compressed.bands[0].metadata['IMAGE_STRUCTURE']['PIXELTYPE'],
            'SIGNEDBYTE')
        if GDAL_VERSION >= (2, 1):
            self.assertIn('Block=40x23', compressed.info)

    def test_raster_warp(self):
        # Create in memory raster
        source = GDALRaster({
            'datatype':
            1,
            'driver':
            'MEM',
            'name':
            'sourceraster',
            'width':
            4,
            'height':
            4,
            'nr_of_bands':
            1,
            'srid':
            3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(16),
                'nodata_value': 255,
            }],
        })

        # Test altering the scale, width, and height of a raster
        data = {
            'scale': [200, -200],
            'width': 2,
            'height': 2,
        }
        target = source.warp(data)
        self.assertEqual(target.width, data['width'])
        self.assertEqual(target.height, data['height'])
        self.assertEqual(target.scale, data['scale'])
        self.assertEqual(target.bands[0].datatype(),
                         source.bands[0].datatype())
        self.assertEqual(target.name, 'sourceraster_copy.MEM')
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(result, [5, 7, 13, 15])

        # Test altering the name and datatype (to float)
        data = {
            'name': '/path/to/targetraster.tif',
            'datatype': 6,
        }
        target = source.warp(data)
        self.assertEqual(target.bands[0].datatype(), 6)
        self.assertEqual(target.name, '/path/to/targetraster.tif')
        self.assertEqual(target.driver.name, 'MEM')
        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        self.assertEqual(result, [
            0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0,
            13.0, 14.0, 15.0
        ])

    def test_raster_warp_nodata_zone(self):
        # Create in memory raster.
        source = GDALRaster({
            'datatype':
            1,
            'driver':
            'MEM',
            'width':
            4,
            'height':
            4,
            'srid':
            3086,
            'origin': (500000, 400000),
            'scale': (100, -100),
            'skew': (0, 0),
            'bands': [{
                'data': range(16),
                'nodata_value': 23,
            }],
        })
        # Warp raster onto a location that does not cover any pixels of the original.
        result = source.warp({'origin': (200000, 200000)}).bands[0].data()
        if numpy:
            result = result.flatten().tolist()
        # The result is an empty raster filled with the correct nodata value.
        self.assertEqual(result, [23] * 16)

    def test_raster_transform(self):
        # Prepare tempfile and nodata value
        rstfile = tempfile.NamedTemporaryFile(suffix='.tif')
        ndv = 99

        # Create in file based raster
        source = GDALRaster({
            'datatype':
            1,
            'driver':
            'tif',
            'name':
            rstfile.name,
            'width':
            5,
            'height':
            5,
            'nr_of_bands':
            1,
            'srid':
            4326,
            'origin': (-5, 5),
            'scale': (2, -2),
            'skew': (0, 0),
            'bands': [{
                'data': range(25),
                'nodata_value': ndv,
            }],
        })

        # Transform raster into srid 4326.
        target = source.transform(3086)

        # Reload data from disk
        target = GDALRaster(target.name)

        self.assertEqual(target.srs.srid, 3086)
        self.assertEqual(target.width, 7)
        self.assertEqual(target.height, 7)
        self.assertEqual(target.bands[0].datatype(),
                         source.bands[0].datatype())
        self.assertAlmostEqual(target.origin[0], 9124842.791079799, 3)
        self.assertAlmostEqual(target.origin[1], 1589911.6476407414, 3)
        self.assertAlmostEqual(target.scale[0], 223824.82664250192, 3)
        self.assertAlmostEqual(target.scale[1], -223824.82664250192, 3)
        self.assertEqual(target.skew, [0, 0])

        result = target.bands[0].data()
        if numpy:
            result = result.flatten().tolist()

        # The reprojection of a raster that spans over a large area
        # skews the data matrix and might introduce nodata values.
        self.assertEqual(result, [
            ndv,
            ndv,
            ndv,
            ndv,
            4,
            ndv,
            ndv,
            ndv,
            ndv,
            2,
            3,
            9,
            ndv,
            ndv,
            ndv,
            1,
            2,
            8,
            13,
            19,
            ndv,
            0,
            6,
            6,
            12,
            18,
            18,
            24,
            ndv,
            10,
            11,
            16,
            22,
            23,
            ndv,
            ndv,
            ndv,
            15,
            21,
            22,
            ndv,
            ndv,
            ndv,
            ndv,
            20,
            ndv,
            ndv,
            ndv,
            ndv,
        ])