def test_unmount_destructor_conflict(self):
        wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \
              "138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \
              "2469482421875 -27.57843813308233))"

        polygon = loads(wkt)

        # sql_filters = ['cloud_cover=0']
        d_start = date(2006, 8, 4)
        d_end = date(2006, 8, 7)
        bounding_box = polygon.bounds
        # sql_filters = ['wrs_row=79']
        landsat_filters = LandsatQueryFilters()
        landsat_filters.wrs_row.set_value(79)
        landsat_filters.acquired.set_range(d_start, True, d_end, True)
        landsat_filters.aoi.set_bounds(*bounding_box)
        rows = self.metadata_service.search(
            SpacecraftID.LANDSAT_5,
            data_filters=landsat_filters)

        rows = list(rows)

        metadata = rows[0]
        landsat = Landsat(metadata)
        vrt = landsat.get_vrt([4])
        storage = Storage("gcp-public-data-landsat")
        landsat_2 = Landsat(metadata)
        vrt = landsat_2.get_vrt([4])
        del landsat
        self.assertTrue(storage.is_mounted(metadata))
    def test_iowa_ndarray(self):
        d_start = date(2016, 4, 4)
        d_end = date(2016, 8, 7)
        bounding_box = self.iowa_polygon.bounds
        # sql_filters = ["cloud_cover<=15"]
        landsat_filters = LandsatQueryFilters()
        landsat_filters.cloud_cover.set_range(end=15, end_inclusive=15)

        landsat_filters.acquired.set_range(d_start, True, d_end, True)
        landsat_filters.aoi.set_bounds(*bounding_box)
        rows = self.metadata_service.search(
            SpacecraftID.LANDSAT_8,
            data_filters=landsat_filters)
        rows = list(rows)
        metadata = rows[0]
        landsat = Landsat(metadata)

        code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
    with np.errstate(divide = 'ignore', invalid = 'ignore'):
        out_ar[:] = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
        out_ar[np.isnan(out_ar)] = 0.0
        out_ar """

        pixel_function_details = FunctionDetails(name="ndvi_numpy", band_definitions=[4, 5], code=code, data_type=DataType.FLOAT32)
        # pixel_function_details = {
        #     "band_numbers": [4, 5],
        #     "function_code": code,
        #     "function_type": "ndvi_numpy",
        #     "data_type": DataType.FLOAT32,
        # }

        band_definitions = [pixel_function_details, 4, 5]

        vrt = landsat.get_vrt(band_definitions)
        ds = gdal.Open(vrt)
        self.assertIsNotNone(ds)

        arr_4 = ds.GetRasterBand(2).ReadAsArray()
        arr_5 = ds.GetRasterBand(3).ReadAsArray()
        arr_ndvi = ds.GetRasterBand(1).ReadAsArray()
        del ds
        del landsat
        print(np.ndarray.max(arr_ndvi))
        print(np.ndarray.min(arr_ndvi))
        self.assertFalse(np.any(np.isinf(arr_ndvi)))
        self.assertIsNotNone(arr_ndvi)

        local_ndvi = self.ndvi_numpy(arr_5, arr_4)

        del arr_4
        del arr_5
        self.assertFalse(np.any(np.isinf(local_ndvi)))

        np.testing.assert_almost_equal(arr_ndvi, local_ndvi)
    def test_vrt_extent(self):
        # GDAL helper functions for generating VRT
        landsat = Landsat(self.metadata_set[0])

        # get a numpy.ndarray from bands for specified imagery
        band_numbers = [Band.RED, Band.GREEN, Band.BLUE]
        scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]]
        vrt = landsat.get_vrt(band_numbers,
                              envelope_boundary=self.taos_shape.bounds)

        self.assertIsNotNone(vrt)
    def test_pixel_ndvi(self):
        """
        http://grindgis.com/blog/vegetation-indices-arcgis
        NDVI = (NIR - RED) / (NIR + RED)
        NDVI = (5 - 4) / (5 + 4)
        :return:
        """
        landsat = Landsat(self.m_metadata)  # , gsurl[2])

        code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
    with np.errstate(divide = 'ignore', invalid = 'ignore'):
        output = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
        output[np.isnan(output)] = 0.0
        # shift range from -1.0-1.0 to 0.0-2.0
        output += 1.0
        # scale up from 0.0-2.0 to 0 to 255 by multiplying by 255/2
        # https://stackoverflow.com/a/1735122/445372
        output *=  65535/2.0
        # https://stackoverflow.com/a/10622758/445372
        # in place type conversion
        out_ar[:] = output.astype(np.int16, copy=False)"""

        pixel_function_details = FunctionDetails(name="ndvi_numpy", band_definitions=[4, 5],
                                                 data_type=DataType.UINT16, code=code)
        vrt = landsat.get_vrt([pixel_function_details, 3, 2])

        with open('ndvi_numpy.vrt', 'r') as myfile:
            data = myfile.read()
            expected = etree.XML(data)
            actual = etree.XML(vrt)
            result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10})
            self.assertTrue(result, message)

        gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', "YES")

        ds = gdal.Open(vrt)
        self.assertIsNotNone(ds)
        arr_ndvi = ds.GetRasterBand(1).ReadAsArray()
        ds = None
        self.assertIsNotNone(arr_ndvi)
        scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]]
        band_definitions = [pixel_function_details, 3, 2]
        nda = landsat.fetch_imagery_array(band_definitions, scale_params)
        self.assertIsNotNone(nda)
 def test_get_file(self):
     d_start = date(2015, 6, 24)
     d_end = date(2016, 6, 24)
     bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
     landsat_filters = LandsatQueryFilters()
     landsat_filters.acquired.set_range(d_start, True, d_end, True)
     landsat_filters.aoi.set_bounds(*bounding_box)
     metadata_service = MetadataService()
     rows = metadata_service.search(SpacecraftID.LANDSAT_8,
                                    data_filters=landsat_filters,
                                    limit=1)
     rows = list(rows)
     metadata = rows[0]
     landsat = Landsat(metadata)
     self.assertIsNotNone(landsat)
     vrt = landsat.get_vrt([4, 3, 2])
     self.assertIsNotNone(vrt)
     dataset = landsat.get_dataset([4, 3, 2], DataType.UINT16)
     self.assertIsNotNone(dataset)
    def test_metadata_extent(self):
        r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json")
        taos_geom = r.json()
        print(taos_geom)

        taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry'])

        metadata_service = MetadataService()
        # sql_filters = ['scene_id="LC80330342017072LGN00"', 'collection_number="PRE"']
        landsat_filters = LandsatQueryFilters()
        landsat_filters.scene_id.set_value("LC80330342017072LGN00")
        landsat_filters.collection_number.set_value("PRE")
        rows = metadata_service.search(
            SpacecraftID.LANDSAT_8,
            data_filters=landsat_filters)
        rows = list(rows)
        self.assertEqual(len(rows), 1)

        metadata = rows[0]

        # GDAL helper functions for generating VRT
        landsat = Landsat(metadata)

        # get a numpy.ndarray from bands for specified imagery
        band_numbers = [Band.RED, Band.GREEN, Band.BLUE]
        scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]]
        vrt = landsat.get_vrt(band_numbers, envelope_boundary=taos_shape.bounds)

        with open('clipped_LC80330342017072LGN00.vrt', 'r') as myfile:
            data = myfile.read()
            expected = etree.XML(data)
            actual = etree.XML(vrt)
            result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10, "xOff": 1e-10, "yOff": 1e-10})
            self.assertTrue(result, message)

        dataset = gdal.Open(vrt)
        geo_transform = dataset.GetGeoTransform()

        # self.assertEqual(geo_transform, raster_metadata.get_geotransform(taos_shape.bounds))
        # self.assertNotEqual(geo_transform, raster_metadata.get_geotransform())

        """
    def test_pixel_1(self):
        metadata = self.m_metadata
        landsat = Landsat(metadata)  # , gsurl[2])

        code = """import numpy as np
def multiply_rounded(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize,
                   raster_ysize, buf_radius, gt, **kwargs):
    factor = float(kwargs['factor'])
    out_ar[:] = np.round_(np.clip(in_ar[0] * factor,0,255))"""

        function_arguments = {"factor": "1.5"}
        pixel_function_details = FunctionDetails(name="multiply_rounded", band_definitions=[2],
                                                 data_type=DataType.FLOAT32, code=code,
                                                 arguments=function_arguments)

        vrt = landsat.get_vrt([pixel_function_details, 3, 2])

        with open('pixel_1.vrt', 'r') as myfile:
            data = myfile.read()
            expected = etree.XML(data)
            actual = etree.XML(vrt)
            result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10})
            self.assertTrue(result, message)
    def test_vrt_not_pre(self):
        d_start = date(2017, 6, 24)
        d_end = date(2017, 9, 24)
        bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999,
                        44.84029065139799)
        # sql_filters = ['collection_number!="PRE"']
        landsat_filter = LandsatQueryFilters()
        landsat_filter.collection_number.set_exclude_value("PRE")
        landsat_filter.acquired.set_range(d_start, True, d_end, True)
        landsat_filter.aoi.set_bounds(*bounding_box)
        rows = self.metadata_service.search(SpacecraftID.LANDSAT_8,
                                            limit=1,
                                            data_filters=landsat_filter)

        rows = list(rows)
        metadata = rows[0]

        landsat = Landsat(metadata)
        self.assertIsNotNone(landsat)
        vrt = landsat.get_vrt([4, 3, 2])
        self.assertIsNotNone(vrt)
        dataset = landsat.get_dataset([4, 3, 2], DataType.UINT16)
        self.assertIsNotNone(dataset)
    def test_vrt(self):
        d_start = date(2015, 6, 24)
        d_end = date(2016, 6, 24)
        bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
        # sql_filters = ['scene_id="LC80400312016103LGN00"']
        landsat_filters = LandsatQueryFilters()
        landsat_filters.scene_id.set_value("LC80400312016103LGN00")
        landsat_filters.acquired.set_range(d_start, True, d_end, True)
        landsat_filters.aoi.set_bounds(*bounding_box)
        rows = self.metadata_service.search(SpacecraftID.LANDSAT_8,
                                            limit=1,
                                            data_filters=landsat_filters)

        rows = list(rows)

        metadata = rows[0]
        landsat = Landsat(metadata)
        vrt = landsat.get_vrt([4, 3, 2])
        with open('test_1.vrt', 'r') as myfile:
            data = myfile.read()
            expected = etree.XML(data)
            actual = etree.XML(vrt)
            result, message = xml_compare(expected, actual)
            self.assertTrue(result, message)

        dataset = gdal.Open(vrt)
        self.assertIsNotNone(dataset)

        ds_band_1 = dataset.GetRasterBand(1)
        self.assertIsNotNone(ds_band_1)
        self.assertEqual(ds_band_1.XSize, 7631)
        ds_band_2 = dataset.GetRasterBand(2)
        self.assertIsNotNone(ds_band_2)
        self.assertEqual(ds_band_2.YSize, 7771)
        ds_band_3 = dataset.GetRasterBand(3)
        self.assertIsNotNone(ds_band_3)
        self.assertEqual(ds_band_3.YSize, 7771)
    def test_landsat5_vrt(self):
        # 5th Place: Lake Eyre Landsat 5 Acquired August 5, 2006
        wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \
              "138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \
              "2469482421875 -27.57843813308233))"

        polygon = loads(wkt)

        # sql_filters = ['cloud_cover=0']
        d_start = date(2006, 8, 4)
        d_end = date(2006, 8, 5)
        bounding_box = polygon.bounds
        landsat_filters = LandsatQueryFilters()
        landsat_filters.wrs_row.set_value(79)
        landsat_filters.acquired.set_range(d_start, True, d_end, True)
        landsat_filters.aoi.set_bounds(*bounding_box)
        rows = self.metadata_service.search(
            SpacecraftID.LANDSAT_5,
            data_filters=landsat_filters)

        rows = list(rows)

        self.assertEqual(len(rows), 1)

        # data structure that contains all fields from Google's Landsat BigQuery Database
        metadata = rows[0]

        # GDAL helper functions for generating VRT
        landsat = Landsat(metadata)
        vrt = landsat.get_vrt([3, 2, 1])

        with open('testlandsat5.vrt', 'r') as myfile:
            data = myfile.read()
            expected = etree.XML(data)
            actual = etree.XML(vrt)
            result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10})
            self.assertTrue(result, message)
    def test_iowa_scaled(self):
        d_start = date(2016, 4, 4)
        d_end = date(2016, 8, 7)
        bounding_box = self.iowa_polygon.bounds
        # sql_filters = ["cloud_cover<=15"]
        landsat_filters = LandsatQueryFilters()
        landsat_filters.cloud_cover.set_range(end=15, end_inclusive=True)

        landsat_filters.acquired.set_range(d_start, True, d_end, True)
        landsat_filters.aoi.set_bounds(*bounding_box)
        rows = self.metadata_service.search(
            SpacecraftID.LANDSAT_8,
            data_filters=landsat_filters)
        rows = list(rows)
        metadata = rows[0]
        landsat = Landsat(metadata)

        code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
    with np.errstate(divide = 'ignore', invalid = 'ignore'):
        factor = float(kwargs['factor'])
        out_ar[:] = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
        out_ar[np.isnan(out_ar)] = 0.0
        # shift range from -1.0-1.0 to 0.0-2.0
        out_ar += 1.0
        # scale up from 0.0-2.0 to 0 to 255 by multiplying by 255/2
        out_ar *= factor/2.0"""

        # pixel_function_details = {
        #     "function_arguments": {"factor": 255},
        #     "band_numbers": [4, Band.NIR],
        #     "function_code": code,
        #     "function_type": "ndvi_numpy",
        #     "data_type": DataType.FLOAT32,
        # }

        pixel_function_details = FunctionDetails(name="ndvi_numpy",
                                                 band_definitions=[4, Band.NIR],
                                                 code=code, arguments={"factor": 255},
                                                 data_type=DataType.FLOAT32)

        band_definitions = [pixel_function_details, Band.RED, 5]

        vrt = landsat.get_vrt(band_definitions)
        ds = gdal.Open(vrt)

        self.assertIsNotNone(ds)

        arr_4 = ds.GetRasterBand(2).ReadAsArray()
        arr_5 = ds.GetRasterBand(3).ReadAsArray()
        arr_ndvi = ds.GetRasterBand(1).ReadAsArray()
        del ds
        del landsat
        print(np.ndarray.max(arr_ndvi))
        # print(np.ndarray.min(arr_ndvi))
        # self.assertFalse(np.any(np.isinf(arr_ndvi)))
        self.assertIsNotNone(arr_ndvi)

        local_ndvi = self.ndvi_numpy(arr_5, arr_4)
        del arr_4
        del arr_5

        local_ndvi += 1.0
        local_ndvi *= float(pixel_function_details.arguments['factor']) / 2.0
        self.assertFalse(np.any(np.isinf(local_ndvi)))

        np.floor(arr_ndvi, out=arr_ndvi)
        np.floor(local_ndvi, out=local_ndvi)
        np.testing.assert_almost_equal(arr_ndvi, local_ndvi, decimal=0)