def test_unmount_destructor_conflict(self):
wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \
"138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \
"2469482421875 -27.57843813308233))"
polygon = loads(wkt)
# sql_filters = ['cloud_cover=0']
d_start = date(2006, 8, 4)
d_end = date(2006, 8, 7)
bounding_box = polygon.bounds
# sql_filters = ['wrs_row=79']
landsat_filters = LandsatQueryFilters()
landsat_filters.wrs_row.set_value(79)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_5,
data_filters=landsat_filters)
rows = list(rows)
metadata = rows[0]
landsat = Landsat(metadata)
vrt = landsat.get_vrt([4])
storage = Storage("gcp-public-data-landsat")
landsat_2 = Landsat(metadata)
vrt = landsat_2.get_vrt([4])
del landsat
self.assertTrue(storage.is_mounted(metadata))
def test_iowa_ndarray(self):
d_start = date(2016, 4, 4)
d_end = date(2016, 8, 7)
bounding_box = self.iowa_polygon.bounds
# sql_filters = ["cloud_cover<=15"]
landsat_filters = LandsatQueryFilters()
landsat_filters.cloud_cover.set_range(end=15, end_inclusive=15)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
metadata = rows[0]
landsat = Landsat(metadata)
code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
with np.errstate(divide = 'ignore', invalid = 'ignore'):
out_ar[:] = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
out_ar[np.isnan(out_ar)] = 0.0
out_ar """
pixel_function_details = FunctionDetails(name="ndvi_numpy", band_definitions=[4, 5], code=code, data_type=DataType.FLOAT32)
# pixel_function_details = {
# "band_numbers": [4, 5],
# "function_code": code,
# "function_type": "ndvi_numpy",
# "data_type": DataType.FLOAT32,
# }
band_definitions = [pixel_function_details, 4, 5]
vrt = landsat.get_vrt(band_definitions)
ds = gdal.Open(vrt)
self.assertIsNotNone(ds)
arr_4 = ds.GetRasterBand(2).ReadAsArray()
arr_5 = ds.GetRasterBand(3).ReadAsArray()
arr_ndvi = ds.GetRasterBand(1).ReadAsArray()
del ds
del landsat
print(np.ndarray.max(arr_ndvi))
print(np.ndarray.min(arr_ndvi))
self.assertFalse(np.any(np.isinf(arr_ndvi)))
self.assertIsNotNone(arr_ndvi)
local_ndvi = self.ndvi_numpy(arr_5, arr_4)
del arr_4
del arr_5
self.assertFalse(np.any(np.isinf(local_ndvi)))
np.testing.assert_almost_equal(arr_ndvi, local_ndvi)
def test_vrt_extent(self):
# GDAL helper functions for generating VRT
landsat = Landsat(self.metadata_set[0])
# get a numpy.ndarray from bands for specified imagery
band_numbers = [Band.RED, Band.GREEN, Band.BLUE]
scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]]
vrt = landsat.get_vrt(band_numbers,
envelope_boundary=self.taos_shape.bounds)
self.assertIsNotNone(vrt)
def test_pixel_ndvi(self):
"""
http://grindgis.com/blog/vegetation-indices-arcgis
NDVI = (NIR - RED) / (NIR + RED)
NDVI = (5 - 4) / (5 + 4)
:return:
"""
landsat = Landsat(self.m_metadata) # , gsurl[2])
code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
with np.errstate(divide = 'ignore', invalid = 'ignore'):
output = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
output[np.isnan(output)] = 0.0
# shift range from -1.0-1.0 to 0.0-2.0
output += 1.0
# scale up from 0.0-2.0 to 0 to 255 by multiplying by 255/2
# https://stackoverflow.com/a/1735122/445372
output *= 65535/2.0
# https://stackoverflow.com/a/10622758/445372
# in place type conversion
out_ar[:] = output.astype(np.int16, copy=False)"""
pixel_function_details = FunctionDetails(name="ndvi_numpy", band_definitions=[4, 5],
data_type=DataType.UINT16, code=code)
vrt = landsat.get_vrt([pixel_function_details, 3, 2])
with open('ndvi_numpy.vrt', 'r') as myfile:
data = myfile.read()
expected = etree.XML(data)
actual = etree.XML(vrt)
result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10})
self.assertTrue(result, message)
gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', "YES")
ds = gdal.Open(vrt)
self.assertIsNotNone(ds)
arr_ndvi = ds.GetRasterBand(1).ReadAsArray()
ds = None
self.assertIsNotNone(arr_ndvi)
scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]]
band_definitions = [pixel_function_details, 3, 2]
nda = landsat.fetch_imagery_array(band_definitions, scale_params)
self.assertIsNotNone(nda)
def test_get_file(self):
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
metadata_service = MetadataService()
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters,
limit=1)
rows = list(rows)
metadata = rows[0]
landsat = Landsat(metadata)
self.assertIsNotNone(landsat)
vrt = landsat.get_vrt([4, 3, 2])
self.assertIsNotNone(vrt)
dataset = landsat.get_dataset([4, 3, 2], DataType.UINT16)
self.assertIsNotNone(dataset)
def test_metadata_extent(self):
r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json")
taos_geom = r.json()
print(taos_geom)
taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry'])
metadata_service = MetadataService()
# sql_filters = ['scene_id="LC80330342017072LGN00"', 'collection_number="PRE"']
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80330342017072LGN00")
landsat_filters.collection_number.set_value("PRE")
rows = metadata_service.search(
SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 1)
metadata = rows[0]
# GDAL helper functions for generating VRT
landsat = Landsat(metadata)
# get a numpy.ndarray from bands for specified imagery
band_numbers = [Band.RED, Band.GREEN, Band.BLUE]
scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]]
vrt = landsat.get_vrt(band_numbers, envelope_boundary=taos_shape.bounds)
with open('clipped_LC80330342017072LGN00.vrt', 'r') as myfile:
data = myfile.read()
expected = etree.XML(data)
actual = etree.XML(vrt)
result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10, "xOff": 1e-10, "yOff": 1e-10})
self.assertTrue(result, message)
dataset = gdal.Open(vrt)
geo_transform = dataset.GetGeoTransform()
# self.assertEqual(geo_transform, raster_metadata.get_geotransform(taos_shape.bounds))
# self.assertNotEqual(geo_transform, raster_metadata.get_geotransform())
"""
def test_pixel_1(self):
metadata = self.m_metadata
landsat = Landsat(metadata) # , gsurl[2])
code = """import numpy as np
def multiply_rounded(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize,
raster_ysize, buf_radius, gt, **kwargs):
factor = float(kwargs['factor'])
out_ar[:] = np.round_(np.clip(in_ar[0] * factor,0,255))"""
function_arguments = {"factor": "1.5"}
pixel_function_details = FunctionDetails(name="multiply_rounded", band_definitions=[2],
data_type=DataType.FLOAT32, code=code,
arguments=function_arguments)
vrt = landsat.get_vrt([pixel_function_details, 3, 2])
with open('pixel_1.vrt', 'r') as myfile:
data = myfile.read()
expected = etree.XML(data)
actual = etree.XML(vrt)
result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10})
self.assertTrue(result, message)
def test_vrt_not_pre(self):
d_start = date(2017, 6, 24)
d_end = date(2017, 9, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999,
44.84029065139799)
# sql_filters = ['collection_number!="PRE"']
landsat_filter = LandsatQueryFilters()
landsat_filter.collection_number.set_exclude_value("PRE")
landsat_filter.acquired.set_range(d_start, True, d_end, True)
landsat_filter.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(SpacecraftID.LANDSAT_8,
limit=1,
data_filters=landsat_filter)
rows = list(rows)
metadata = rows[0]
landsat = Landsat(metadata)
self.assertIsNotNone(landsat)
vrt = landsat.get_vrt([4, 3, 2])
self.assertIsNotNone(vrt)
dataset = landsat.get_dataset([4, 3, 2], DataType.UINT16)
self.assertIsNotNone(dataset)
def test_vrt(self):
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
# sql_filters = ['scene_id="LC80400312016103LGN00"']
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80400312016103LGN00")
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(SpacecraftID.LANDSAT_8,
limit=1,
data_filters=landsat_filters)
rows = list(rows)
metadata = rows[0]
landsat = Landsat(metadata)
vrt = landsat.get_vrt([4, 3, 2])
with open('test_1.vrt', 'r') as myfile:
data = myfile.read()
expected = etree.XML(data)
actual = etree.XML(vrt)
result, message = xml_compare(expected, actual)
self.assertTrue(result, message)
dataset = gdal.Open(vrt)
self.assertIsNotNone(dataset)
ds_band_1 = dataset.GetRasterBand(1)
self.assertIsNotNone(ds_band_1)
self.assertEqual(ds_band_1.XSize, 7631)
ds_band_2 = dataset.GetRasterBand(2)
self.assertIsNotNone(ds_band_2)
self.assertEqual(ds_band_2.YSize, 7771)
ds_band_3 = dataset.GetRasterBand(3)
self.assertIsNotNone(ds_band_3)
self.assertEqual(ds_band_3.YSize, 7771)
def test_landsat5_vrt(self):
# 5th Place: Lake Eyre Landsat 5 Acquired August 5, 2006
wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \
"138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \
"2469482421875 -27.57843813308233))"
polygon = loads(wkt)
# sql_filters = ['cloud_cover=0']
d_start = date(2006, 8, 4)
d_end = date(2006, 8, 5)
bounding_box = polygon.bounds
landsat_filters = LandsatQueryFilters()
landsat_filters.wrs_row.set_value(79)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_5,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 1)
# data structure that contains all fields from Google's Landsat BigQuery Database
metadata = rows[0]
# GDAL helper functions for generating VRT
landsat = Landsat(metadata)
vrt = landsat.get_vrt([3, 2, 1])
with open('testlandsat5.vrt', 'r') as myfile:
data = myfile.read()
expected = etree.XML(data)
actual = etree.XML(vrt)
result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10})
self.assertTrue(result, message)
def test_iowa_scaled(self):
d_start = date(2016, 4, 4)
d_end = date(2016, 8, 7)
bounding_box = self.iowa_polygon.bounds
# sql_filters = ["cloud_cover<=15"]
landsat_filters = LandsatQueryFilters()
landsat_filters.cloud_cover.set_range(end=15, end_inclusive=True)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
metadata = rows[0]
landsat = Landsat(metadata)
code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
with np.errstate(divide = 'ignore', invalid = 'ignore'):
factor = float(kwargs['factor'])
out_ar[:] = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
out_ar[np.isnan(out_ar)] = 0.0
# shift range from -1.0-1.0 to 0.0-2.0
out_ar += 1.0
# scale up from 0.0-2.0 to 0 to 255 by multiplying by 255/2
out_ar *= factor/2.0"""
# pixel_function_details = {
# "function_arguments": {"factor": 255},
# "band_numbers": [4, Band.NIR],
# "function_code": code,
# "function_type": "ndvi_numpy",
# "data_type": DataType.FLOAT32,
# }
pixel_function_details = FunctionDetails(name="ndvi_numpy",
band_definitions=[4, Band.NIR],
code=code, arguments={"factor": 255},
data_type=DataType.FLOAT32)
band_definitions = [pixel_function_details, Band.RED, 5]
vrt = landsat.get_vrt(band_definitions)
ds = gdal.Open(vrt)
self.assertIsNotNone(ds)
arr_4 = ds.GetRasterBand(2).ReadAsArray()
arr_5 = ds.GetRasterBand(3).ReadAsArray()
arr_ndvi = ds.GetRasterBand(1).ReadAsArray()
del ds
del landsat
print(np.ndarray.max(arr_ndvi))
# print(np.ndarray.min(arr_ndvi))
# self.assertFalse(np.any(np.isinf(arr_ndvi)))
self.assertIsNotNone(arr_ndvi)
local_ndvi = self.ndvi_numpy(arr_5, arr_4)
del arr_4
del arr_5
local_ndvi += 1.0
local_ndvi *= float(pixel_function_details.arguments['factor']) / 2.0
self.assertFalse(np.any(np.isinf(local_ndvi)))
np.floor(arr_ndvi, out=arr_ndvi)
np.floor(local_ndvi, out=local_ndvi)
np.testing.assert_almost_equal(arr_ndvi, local_ndvi, decimal=0)