def test_aws_file_path(self): # PRE s3://landsat-pds/L8/139/045/LC81390452014295LGN00/ # non-PRE s3://landsat-pds/c1/L8/139/045/LC08_L1TP_139045_20170304_20170316_01_T1/ metadataservice = MetadataService() start_date = datetime.datetime.strptime('14295', '%y%j').date() landsat_filters = LandsatQueryFilters() landsat_filters.wrs_path.set_value(139) landsat_filters.wrs_row.set_value(45) landsat_filters.acquired.set_range(start_date, True, start_date, True) landsat_filters.collection_number.set_value("PRE") rows = metadataservice.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filters) # turn gernator into list metadata_set = list(rows) self.assertEqual(len(metadata_set), 1) metadata = metadata_set[0] self.assertEqual(metadata.get_aws_file_path(), "/imagery/L8/139/045/LC81390452014295LGN00") landsat_filters = LandsatQueryFilters() landsat_filters.wrs_path.set_value(139) landsat_filters.wrs_row.set_value(45) # landsat_filters.collection_number.set_exclude_value("PRE") landsat_filters.acquired.set_range(start=date(2017, 3, 4), end=date(2017, 3, 4)) rows = metadataservice.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filters) metadata_set = list(rows) self.assertEqual(len(metadata_set), 2) metadata = metadata_set[0] self.assertEqual(metadata.get_aws_file_path(), "/imagery/c1/L8/139/045/LC08_L1TP_139045_20170304_20170316_01_T1") metadata = metadata_set[1] self.assertEqual(metadata.get_aws_file_path(), "/imagery/L8/139/045/LC81390452017063LGN00")
def test_cloud_cover(self): metadata_service = MetadataService() landsat_filters = LandsatQueryFilters() landsat_filters.cloud_cover.set_value(0) # landsat_filters.scene_id.set_value("LC80390332016208LGN00") # sql_filters = ['cloud_cover=0'] d_start = date(2015, 6, 24) d_end = date(2016, 6, 24) bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799) landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = metadata_service.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filters) rows = list(rows) self.assertEqual(len(rows), 10) for row in rows: self.assertEqual(row.spacecraft_id.name, SpacecraftID.LANDSAT_8.name) d_actual = datetime.datetime.strptime(row.date_acquired, '%Y-%m-%d').date() self.assertLessEqual(d_actual, d_end) self.assertGreaterEqual(d_actual, d_start) test_box = row.bounds self.assertTrue( (bounding_box[0] < test_box[2] < bounding_box[2]) or (bounding_box[0] < test_box[0] < bounding_box[2])) self.assertTrue( (bounding_box[1] < test_box[3] < bounding_box[3]) or (bounding_box[1] < test_box[1] < bounding_box[3]))
def test_end_date(self): r = requests.get( "https://raw.githubusercontent.com/johan/world.geo.json/master/countries/BEL.geo.json" ) area_geom = r.json() area_shape = shapely.geometry.shape( area_geom['features'][0]['geometry']) # gs://gcp-public-data-landsat/LC08/PRE/044/034/LC80440342016259LGN00/ metadata_service = MetadataService() d = date(2016, 6, 24) landsat_filter = LandsatQueryFilters() landsat_filter.acquired.set_range(end=d, end_inclusive=True) landsat_filter.acquired.sort_by(epl_imagery_pb2.DESCENDING) landsat_filter.aoi.set_bounds(*area_shape.bounds) rows = metadata_service.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filter) rows = list(rows) self.assertEqual(len(rows), 10) d_previous = d for row in rows: self.assertEqual(row.spacecraft_id.name, SpacecraftID.LANDSAT_8.name) d_actual = datetime.datetime.strptime(row.date_acquired, '%Y-%m-%d').date() self.assertLessEqual(d_actual, d_previous) d_previous = d_actual
def test_start_date(self): # gs://gcp-public-data-landsat/LC08/PRE/044/034/LC80440342016259LGN00/ metadata_service = MetadataService() d = date(2016, 6, 24) landsat_filters = LandsatQueryFilters() landsat_filters.acquired.set_range(d, True) landsat_filters.wrs_path.set_value(125) landsat_filters.wrs_row.set_value(49) landsat_filters.acquired.sort_by(epl_imagery_pb2.ASCENDING) landsat_filters.acquired.set_range(end=d, end_inclusive=True) rows = metadata_service.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filters) rows = list(rows) self.assertEqual(len(rows), 10) d_previous = datetime.datetime.strptime("1945-01-01", '%Y-%m-%d').date() for row in rows: self.assertEqual(row.spacecraft_id.name, SpacecraftID.LANDSAT_8.name) d_actual = datetime.datetime.strptime(row.date_acquired, '%Y-%m-%d').date() # test Order by self.assertGreaterEqual(d_actual, d_previous) d_previous = d_actual
def test_polygon_boundary(self): d_start = date(2017, 3, 12) # 2017-03-12 d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive r = requests.get( "https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json" ) taos_geom = r.json() taos_shape = shapely.geometry.shape( taos_geom['features'][0]['geometry']) metadata_service = MetadataService() landsat_filters = LandsatQueryFilters() landsat_filters.collection_number.set_value("PRE") landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_geometry(taos_shape.wkb) # landsat_filters.geometry_bag.geometry_binaries.append(taos_shape.wkb) metadata_rows = metadata_service.search(SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filters) # mounted directory in docker container metadata_set = [] for row in metadata_rows: metadata_set.append(row) self.assertEqual(len(metadata_set), 2)
def setUp(self): metadata_service = MetadataService() d_start = date(2015, 6, 24) d_end = date(2016, 6, 24) bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799) landsat_filters = LandsatQueryFilters() landsat_filters.scene_id.set_value("LC80400312016103LGN00") rows = metadata_service.search(SpacecraftID.LANDSAT_8, limit=1, data_filters=landsat_filters) rows = list(rows) self.m_row_data = rows[0] wkt_iowa = "POLYGON((-93.76075744628906 42.32707774458643,-93.47854614257812 42.32707774458643," \ "-93.47854614257812 42.12674735753131,-93.76075744628906 42.12674735753131," \ "-93.76075744628906 42.32707774458643))" self.iowa_polygon = loads(wkt_iowa) gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', "YES") d_start = date(2017, 3, 12) # 2017-03-12 d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive landsat_filters = LandsatQueryFilters() landsat_filters.collection_number.set_exclude_value("PRE") landsat_filters.acquired.set_range(start=d_start, end=d_end) landsat_filters.aoi.set_bounds(*self.taos_shape.bounds) landsat_filters.data_type.set_exclude_value('L1GT') rows = self.metadata_service.search( SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filters) for row in rows: self.metadata_set.append(row)
def test_metatdata_file_list(self): wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \ "138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \ "2469482421875 -27.57843813308233))" polygon = loads(wkt) metadata_service = MetadataService() # sql_filters = ['cloud_cover=0'] d_start = date(2006, 8, 4) d_end = date(2006, 8, 5) bounding_box = polygon.bounds # sql_filters = ['wrs_row=79'] landsat_filters = LandsatQueryFilters() # landsat_filters.wrs_path.set_value(125) landsat_filters.wrs_row.set_value(79) landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = metadata_service.search( SpacecraftID.LANDSAT_5, data_filters=landsat_filters) rows = list(rows) metadata = rows[0] self.assertEqual(len(metadata.get_file_list()), 0)
def test_scene_id(self): landsat_filters = LandsatQueryFilters() landsat_filters.scene_id.set_value("LC80390332016208LGN00") metadata_service = MetadataService() rows = metadata_service.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filters) rows = list(rows) self.assertEqual(len(rows), 1)
def test_where_start(self): # sql_filters = ['scene_id="LC80270312016188LGN00"'] landsat_filters = LandsatQueryFilters() landsat_filters.scene_id.set_value("LC80270312016188LGN00") metadata_service = MetadataService() metadata_rows = metadata_service.search( SpacecraftID.UNKNOWN_SPACECRAFT, data_filters=landsat_filters) metadata_set = list(metadata_rows) self.assertEqual(1, len(metadata_set))
def test_delete_storage(self): metadata_service = MetadataService() d_start = date(2015, 6, 24) d_end = date(2016, 6, 24) bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799) landsat_filters = LandsatQueryFilters() landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = metadata_service.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filters, limit=1) rows = list(rows) metadata = rows[0]
def test_all_sat_data(self): metadata_service = MetadataService() landsat_filters = LandsatQueryFilters() landsat_filters.cloud_cover.set_value(0) d_start = date(2004, 6, 24) d_end = date(2008, 6, 24) bounding_box = (-115.927734375, 34.52466147177172, -114.31054687499999, 35.84029065139799) landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = metadata_service.search( data_filters=landsat_filters) rows = list(rows) first_item = rows[0] self.assertEqual(len(rows), 10) rows = metadata_service.search( satellite_id=SpacecraftID.UNKNOWN_SPACECRAFT, data_filters=landsat_filters) rows = list(rows) other_item = rows[0] self.assertEqual(len(rows), 10) self.assertEqual(first_item.scene_id, other_item.scene_id)
def test_alaskan_aleutians(self): # wkt = "MULTIPOLYGON (((40 40, 20 45, 45 30, 40 40)), ((-172 53, 175 53, 175 48, -172 48, -172 53)), ((20 35, 45 20, 30 5, 10 10, 10 30, 20 35), (30 20, 20 25, 20 15, 30 20)))" wkt = "POLYGON((-172 53, 175 53, 175 48, -172 48, -172 53))" islands_shape = loads(wkt) self.assertFalse(islands_shape.exterior.is_ccw) query_filter = LandsatQueryFilters() query_filter.aoi.set_geometry(islands_shape.wkb) # query_filter.geometry_bag.geometry_binaries.append(islands_shape.wkb) metadata_service = MetadataService() rows = metadata_service.search(SpacecraftID.LANDSAT_8, limit=10, data_filters=query_filter) stuff = list(rows) self.assertEqual(10, len(stuff))
def test_no_bounding_box(self): d_start = date(2003, 4, 4) d_end = date(2003, 4, 7) landsat_filters = LandsatQueryFilters() landsat_filters.wrs_path.set_value(125) landsat_filters.wrs_row.set_value(49) # sql_filters = ['wrs_row=49', 'wrs_path=125'] metadata_service = MetadataService() landsat_filters.acquired.set_range(d_start, True, d_end, True) rows = metadata_service.search( satellite_id=None, data_filters=landsat_filters) rows = list(rows) self.assertEqual(len(rows), 3)
def test_one_day(self): # gs://gcp-public-data-landsat/LC08/PRE/044/034/LC80440342016259LGN00/ metadata_service = MetadataService() d = date(2016, 6, 24) landsat_filter = LandsatQueryFilters() landsat_filter.acquired.set_value(d) rows = metadata_service.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filter) rows = list(rows) self.assertEqual(len(rows), 10) for row in rows: self.assertEqual(row.spacecraft_id.name, SpacecraftID.LANDSAT_8.name) d_actual = datetime.datetime.strptime(row.date_acquired, '%Y-%m-%d').date() self.assertEqual(d_actual, d)
def test_google_aws_mismatch(self): metadata_service = MetadataService() d_start = date(2017, 6, 24) d_end = date(2017, 9, 24) bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799) landsat_filters = LandsatQueryFilters() landsat_filters.collection_number.set_exclude_value("PRE") landsat_filters.acquired.set_range(start=d_start, end=d_end) landsat_filters.aoi.set_bounds(*bounding_box) rows = metadata_service.search(SpacecraftID.LANDSAT_8, limit=2, data_filters=landsat_filters) # generator to list rows = list(rows) metadata = rows[0] # self.assertEqual('LC08_L1GT_135215_20170916_20170916_01_RT', metadata.product_id) self.assertEqual('RT', metadata.collection_category)
def test_get_file(self): d_start = date(2015, 6, 24) d_end = date(2016, 6, 24) bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799) landsat_filters = LandsatQueryFilters() landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) metadata_service = MetadataService() rows = metadata_service.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filters, limit=1) rows = list(rows) metadata = rows[0] landsat = Landsat(metadata) self.assertIsNotNone(landsat) vrt = landsat.get_vrt([4, 3, 2]) self.assertIsNotNone(vrt) dataset = landsat.get_dataset([4, 3, 2], DataType.UINT16) self.assertIsNotNone(dataset)
def test_metadata_extent(self): r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json") taos_geom = r.json() print(taos_geom) taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry']) metadata_service = MetadataService() # sql_filters = ['scene_id="LC80330342017072LGN00"', 'collection_number="PRE"'] landsat_filters = LandsatQueryFilters() landsat_filters.scene_id.set_value("LC80330342017072LGN00") landsat_filters.collection_number.set_value("PRE") rows = metadata_service.search( SpacecraftID.LANDSAT_8, data_filters=landsat_filters) rows = list(rows) self.assertEqual(len(rows), 1) metadata = rows[0] # GDAL helper functions for generating VRT landsat = Landsat(metadata) # get a numpy.ndarray from bands for specified imagery band_numbers = [Band.RED, Band.GREEN, Band.BLUE] scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]] vrt = landsat.get_vrt(band_numbers, envelope_boundary=taos_shape.bounds) with open('clipped_LC80330342017072LGN00.vrt', 'r') as myfile: data = myfile.read() expected = etree.XML(data) actual = etree.XML(vrt) result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10, "xOff": 1e-10, "yOff": 1e-10}) self.assertTrue(result, message) dataset = gdal.Open(vrt) geo_transform = dataset.GetGeoTransform() # self.assertEqual(geo_transform, raster_metadata.get_geotransform(taos_shape.bounds)) # self.assertNotEqual(geo_transform, raster_metadata.get_geotransform()) """
def test_storage_create(self): metadata_service = MetadataService() d_start = date(2015, 6, 24) d_end = date(2016, 6, 24) bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799) landsat_filters = LandsatQueryFilters() landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = metadata_service.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filters, limit=1) rows = list(rows) metadata = rows[0] storage = Storage(metadata.bucket_name) metadata = rows[0] self.assertTrue(storage.mount_sub_folder(metadata, "generic")) self.assertTrue(storage.unmount_sub_folder(metadata, "generic"))
def test_bounds(self): metadata_service = MetadataService() landsat_filters = LandsatQueryFilters() landsat_filters.scene_id.set_value("LC80330342017072LGN00") landsat_filters.collection_number.set_value("PRE") rows = metadata_service.search( SpacecraftID.LANDSAT_8, data_filters=landsat_filters) rows = list(rows) self.assertEqual(len(rows), 1) metadata = rows[0] bands = [Band.RED, Band.BLUE, Band.GREEN] band_map = BandMap(SpacecraftID.LANDSAT_8) raster_metadata = RasterMetadata() storage = Storage() storage.mount_sub_folder(metadata) for band in bands: band_number = band_map.get_number(band) raster_metadata.add_metadata(band_number, metadata) boundary = raster_metadata.bounds self.assertIsNotNone(boundary) r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json") taos_geom = r.json() taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry']) clipped_raster = raster_metadata.calculate_clipped(taos_shape.bounds, pyproj.Proj(init='epsg:4326')) self.assertIsNotNone(clipped_raster.bounds) big_box = shapely.geometry.box(*boundary) small_box = shapely.geometry.box(*clipped_raster.bounds) self.assertTrue(big_box.contains(small_box))
def test_colorado(self): r = requests.get( "https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/CO/Boulder.geo.json" ) boulder_geom = r.json() boulder_shape = shapely.geometry.shape( boulder_geom['features'][0]['geometry']) d_start = date(2017, 3, 1) # 2017-03-12 d_end = date(2017, 3, 19) # epl api is inclusive # PRE is a collection type that specifies certain QA standards landsat_filter = LandsatQueryFilters() landsat_filter.collection_number.set_value("PRE") landsat_filter.acquired.set_range(d_start, True, d_end, True) landsat_filter.aoi.set_bounds(*boulder_shape.bounds) # search the satellite metadata for images of Taos withing the given date range metadata_service = MetadataService() rows = metadata_service.search(SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filter) self.assertTrue(2, len(list(rows)))
def test_belgium(self): r = requests.get( "https://raw.githubusercontent.com/johan/world.geo.json/master/countries/BEL.geo.json" ) area_geom = r.json() area_shape = shapely.geometry.shape( area_geom['features'][0]['geometry']) d_start = date(2017, 1, 1) # 2017-03-12 d_end = date(2017, 5, 19) # epl api is inclusive belgium_filter = LandsatQueryFilters() # PRE is a collection type that specifies certain QA standards belgium_filter.collection_number.set_value("PRE") belgium_filter.cloud_cover.set_range(end=15, end_inclusive=False) belgium_filter.acquired.set_range(start=d_start, end=d_end) belgium_filter.aoi.set_bounds(*area_shape.bounds) # search the satellite metadata for images of Belgium withing the given date range metadata_service = MetadataService() rows = metadata_service.search(SpacecraftID.LANDSAT_8, limit=20, data_filters=belgium_filter)
class TestRasterMetadata(unittest.TestCase): base_mount_path = '/imagery' metadata_service = None def setUp(self): self.metadata_service = MetadataService() def test_add_metadata_error(self): d_start = date(2015, 6, 24) d_end = date(2016, 6, 24) bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799) landsat_filters = LandsatQueryFilters() landsat_filters.data_type.set_value("L1T") landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = self.metadata_service.search(SpacecraftID.LANDSAT_8, limit=2, data_filters=landsat_filters) rows = list(rows) metadata_1 = rows[0] metadata_2 = rows[1] bands = [Band.RED, Band.BLUE, Band.GREEN] band_map = BandMap(SpacecraftID.LANDSAT_8) raster_metadata = RasterMetadata() storage = Storage() storage.mount_sub_folder(metadata_1) storage.mount_sub_folder(metadata_2) second = False for band in bands: band_number = band_map.get_number(band) if second: self.assertRaises(Exception, lambda: raster_metadata.add_metadata(band_number, metadata_2)) raster_metadata.add_metadata(band_number, metadata_1) second = True # @unittest.skip("changed how bounds are queried") def test_bounds(self): metadata_service = MetadataService() landsat_filters = LandsatQueryFilters() landsat_filters.scene_id.set_value("LC80330342017072LGN00") landsat_filters.collection_number.set_value("PRE") rows = metadata_service.search( SpacecraftID.LANDSAT_8, data_filters=landsat_filters) rows = list(rows) self.assertEqual(len(rows), 1) metadata = rows[0] bands = [Band.RED, Band.BLUE, Band.GREEN] band_map = BandMap(SpacecraftID.LANDSAT_8) raster_metadata = RasterMetadata() storage = Storage() storage.mount_sub_folder(metadata) for band in bands: band_number = band_map.get_number(band) raster_metadata.add_metadata(band_number, metadata) boundary = raster_metadata.bounds self.assertIsNotNone(boundary) r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json") taos_geom = r.json() taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry']) clipped_raster = raster_metadata.calculate_clipped(taos_shape.bounds, pyproj.Proj(init='epsg:4326')) self.assertIsNotNone(clipped_raster.bounds) big_box = shapely.geometry.box(*boundary) small_box = shapely.geometry.box(*clipped_raster.bounds) self.assertTrue(big_box.contains(small_box)) def test_metadata_extent(self): r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json") taos_geom = r.json() print(taos_geom) taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry']) metadata_service = MetadataService() # sql_filters = ['scene_id="LC80330342017072LGN00"', 'collection_number="PRE"'] landsat_filters = LandsatQueryFilters() landsat_filters.scene_id.set_value("LC80330342017072LGN00") landsat_filters.collection_number.set_value("PRE") rows = metadata_service.search( SpacecraftID.LANDSAT_8, data_filters=landsat_filters) rows = list(rows) self.assertEqual(len(rows), 1) metadata = rows[0] # GDAL helper functions for generating VRT landsat = Landsat(metadata) # get a numpy.ndarray from bands for specified imagery band_numbers = [Band.RED, Band.GREEN, Band.BLUE] scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]] vrt = landsat.get_vrt(band_numbers, envelope_boundary=taos_shape.bounds) with open('clipped_LC80330342017072LGN00.vrt', 'r') as myfile: data = myfile.read() expected = etree.XML(data) actual = etree.XML(vrt) result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10, "xOff": 1e-10, "yOff": 1e-10}) self.assertTrue(result, message) dataset = gdal.Open(vrt) geo_transform = dataset.GetGeoTransform() # self.assertEqual(geo_transform, raster_metadata.get_geotransform(taos_shape.bounds)) # self.assertNotEqual(geo_transform, raster_metadata.get_geotransform()) """
class TestLandsat(unittest.TestCase): base_mount_path = '/imagery' metadata_service = None metadata_set = [] r = requests.get( "https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json" ) taos_geom = r.json() taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry']) def setUp(self): d_start = date(2017, 3, 12) # 2017-03-12 d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive self.metadata_service = MetadataService() landsat_filters = LandsatQueryFilters() landsat_filters.collection_number.set_value("PRE") landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*self.taos_shape.bounds) metadata_rows = self.metadata_service.search( SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filters) # mounted directory in docker container base_mount_path = '/imagery' for row in metadata_rows: self.metadata_set.append(row) # TODO test PRE rejection # TODO test date range rejection # TODO test Satellite Rejection def test_vrt_not_pre(self): d_start = date(2017, 6, 24) d_end = date(2017, 9, 24) bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799) # sql_filters = ['collection_number!="PRE"'] landsat_filter = LandsatQueryFilters() landsat_filter.collection_number.set_exclude_value("PRE") landsat_filter.acquired.set_range(d_start, True, d_end, True) landsat_filter.aoi.set_bounds(*bounding_box) rows = self.metadata_service.search(SpacecraftID.LANDSAT_8, limit=1, data_filters=landsat_filter) rows = list(rows) metadata = rows[0] landsat = Landsat(metadata) self.assertIsNotNone(landsat) vrt = landsat.get_vrt([4, 3, 2]) self.assertIsNotNone(vrt) dataset = landsat.get_dataset([4, 3, 2], DataType.UINT16) self.assertIsNotNone(dataset) def test_pixel_function_vrt_1(self): utah_box = (-112.66342163085938, 37.738141282210385, -111.79824829101562, 38.44821130413263) d_start = date(2016, 7, 20) d_end = date(2016, 7, 28) landsat_filter = LandsatQueryFilters() landsat_filter.collection_number.set_value("PRE") landsat_filter.cloud_cover.set_range( end=5, end_inclusive=True ) #landsat_filter.cloud_cover.set_range_end(5, True) landsat_filter.acquired.set_range(d_start, True, d_end, True) landsat_filter.aoi.set_bounds(*utah_box) rows = self.metadata_service.search(SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filter) rows = list(rows) self.assertEqual(0, len(rows)) d_end = date(2016, 8, 28) landsat_filter = LandsatQueryFilters() landsat_filter.collection_number.set_value("PRE") landsat_filter.cloud_cover.set_range( end=5, end_inclusive=False ) #landsat_filter.cloud_cover.set_range_end(5, False) landsat_filter.acquired.set_range( end=d_end, end_inclusive=True ) #landsat_filter.acquired.set_range_end(d_end, True) landsat_filter.acquired.sort_by(epl_imagery_pb2.DESCENDING) landsat_filter.aoi.set_bounds(*utah_box) rows = self.metadata_service.search(SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filter) rows = list(rows) self.assertEqual(len(rows), 10) # metadata_row = ['LC80390332016208LGN00', '', 'LANDSAT_8', 'OLI_TIRS', '2016-07-26', # '2016-07-26T18:14:46.9465460Z', 'PRE', 'N/A', 'L1T', 39, 33, 1.69, # 39.96962, 37.81744, -115.27267, -112.56732, 1070517542, # 'gs://gcp-public-data-landsat/LC08/PRE/039/033/LC80390332016208LGN00'] metadata = rows[0] # GDAL helper functions for generating VRT landsat = Landsat([metadata]) # get a numpy.ndarray from bands for specified imagery band_numbers = [4, 3, 2] scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]] nda = landsat.fetch_imagery_array(band_numbers, scale_params) self.assertEqual(nda.shape, (3876, 3806, 3)) def test_band_enum(self): self.assertTrue(True) d_start = date(2016, 7, 20) d_end = date(2016, 7, 28) landsat_filter = LandsatQueryFilters() landsat_filter.scene_id.set_value("LC80390332016208LGN00") landsat_filter.acquired.set_range(d_start, True, d_end, True) rows = self.metadata_service.search(SpacecraftID.LANDSAT_8, limit=1, data_filters=landsat_filter) rows = list(rows) metadata = rows[0] landsat = Landsat(metadata) scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]] # nda = landsat.__get_ndarray(band_numbers, metadata, scale_params) nda = landsat.fetch_imagery_array([Band.RED, Band.GREEN, Band.BLUE], scale_params, spatial_resolution_m=240) self.assertIsNotNone(nda) nda2 = landsat.fetch_imagery_array([4, 3, 2], scale_params, spatial_resolution_m=240) np.testing.assert_almost_equal(nda, nda2) # 'scene_id': 'LC80390332016208LGN00' def test_vrt_extent(self): # GDAL helper functions for generating VRT landsat = Landsat(self.metadata_set[0]) # get a numpy.ndarray from bands for specified imagery band_numbers = [Band.RED, Band.GREEN, Band.BLUE] scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]] vrt = landsat.get_vrt(band_numbers, envelope_boundary=self.taos_shape.bounds) self.assertIsNotNone(vrt) def test_cutline(self): # GDAL helper functions for generating VRT landsat = Landsat(self.metadata_set[0]) # get a numpy.ndarray from bands for specified imagery band_numbers = [Band.RED, Band.GREEN, Band.BLUE] scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]] nda = landsat.fetch_imagery_array(band_numbers, scale_params, self.taos_shape.wkb, spatial_resolution_m=480) self.assertIsNotNone(nda) # TODO needs shape test def test_mosaic(self): # GDAL helper functions for generating VRT landsat = Landsat(self.metadata_set) # get a numpy.ndarray from bands for specified imagery band_numbers = [Band.RED, Band.GREEN, Band.BLUE] scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]] nda = landsat.fetch_imagery_array( band_numbers, scale_params, envelope_boundary=self.taos_shape.bounds) self.assertIsNotNone(nda) self.assertEqual((1804, 1295, 3), nda.shape) # TODO needs shape test def test_mosaic_cutline(self): # GDAL helper functions for generating VRT landsat = Landsat(self.metadata_set) # get a numpy.ndarray from bands for specified imagery # 'nir', 'swir1', 'swir2' band_numbers = [Band.NIR, Band.SWIR1, Band.SWIR2] scaleParams = [[0.0, 40000.0], [0.0, 40000.0], [0.0, 40000.0]] nda = landsat.fetch_imagery_array( band_numbers, scaleParams, polygon_boundary_wkb=self.taos_shape.wkb) self.assertIsNotNone(nda) self.assertEqual((1804, 1295, 3), nda.shape) def test_polygon_wkb_metadata(self): d_start = date(2017, 3, 12) # 2017-03-12 d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive self.metadata_service = MetadataService() landsat_filters = LandsatQueryFilters() landsat_filters.collection_number.set_value("PRE") landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_geometry(self.taos_shape.wkb) # landsat_filters.geometry_bag.geometry_binaries.append(self.taos_shape.wkb) metadata_rows = self.metadata_service.search( SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filters) metadata_set = [] for row in metadata_rows: metadata_set.append(row) landsat = Landsat(metadata_set) band_numbers = [Band.NIR, Band.SWIR1, Band.SWIR2] scaleParams = [[0.0, 40000.0], [0.0, 40000.0], [0.0, 40000.0]] nda = landsat.fetch_imagery_array( band_numbers, scaleParams, polygon_boundary_wkb=self.taos_shape.wkb) self.assertIsNotNone(nda) self.assertEqual((1804, 1295, 3), nda.shape) def test_mosaic_mem_error(self): landsat = Landsat(self.metadata_set) # get a numpy.ndarray from bands for specified imagery band_numbers = [Band.RED, Band.GREEN, Band.BLUE] scaleParams = [[0.0, 40000], [0.0, 40000], [0.0, 40000]] nda = landsat.fetch_imagery_array( band_numbers, scaleParams, envelope_boundary=self.taos_shape.bounds) self.assertIsNotNone(nda) # GDAL helper functions for generating VRT landsat = Landsat(self.metadata_set) self.assertEqual((1804, 1295, 3), nda.shape) # get a numpy.ndarray from bands for specified imagery # 'nir', 'swir1', 'swir2' band_numbers = [Band.NIR, Band.SWIR1, Band.SWIR2] scaleParams = [[0.0, 40000.0], [0.0, 40000.0], [0.0, 40000.0]] nda = landsat.fetch_imagery_array( band_numbers, scaleParams, polygon_boundary_wkb=self.taos_shape.wkb) self.assertIsNotNone(nda) self.assertEqual((1804, 1295, 3), nda.shape) def test_datatypes(self): landsat = Landsat(self.metadata_set) # get a numpy.ndarray from bands for specified imagery band_numbers = [Band.RED, Band.GREEN, Band.BLUE] scaleParams = [[0.0, 40000], [0.0, 40000], [0.0, 40000]] for data_type in DataType: if data_type is DataType.UNKNOWN_GDAL: continue nda = landsat.fetch_imagery_array( band_numbers, scaleParams, envelope_boundary=self.taos_shape.bounds, output_type=data_type, spatial_resolution_m=240) self.assertIsNotNone(nda) self.assertGreaterEqual(data_type.range_max, nda.max()) self.assertLessEqual(data_type.range_min, nda.min()) def test_vrt_with_alpha(self): landsat = Landsat(self.metadata_set) # get a numpy.ndarray from bands for specified imagery band_numbers = [Band.RED, Band.GREEN, Band.BLUE, Band.ALPHA] scaleParams = [[0.0, 40000], [0.0, 40000], [0.0, 40000]] nda = landsat.fetch_imagery_array( band_numbers, scaleParams, envelope_boundary=self.taos_shape.bounds, output_type=DataType.UINT16, spatial_resolution_m=120) self.assertIsNotNone(nda) def test_rastermetadata_cache(self): # GDAL helper functions for generating VRT landsat = Landsat(self.metadata_set) # get a numpy.ndarray from bands for specified imagery # 'nir', 'swir1', 'swir2' band_numbers = [Band.NIR, Band.SWIR1, Band.SWIR2] scaleParams = [[0.0, 40000.0], [0.0, 40000.0], [0.0, 40000.0]] nda = landsat.fetch_imagery_array( band_numbers, scaleParams, polygon_boundary_wkb=self.taos_shape.wkb, spatial_resolution_m=120) self.assertIsNotNone(nda) self.assertEqual((902, 648, 3), nda.shape) band_numbers = [Band.RED, Band.BLUE, Band.GREEN] nda = landsat.fetch_imagery_array( band_numbers, scaleParams, polygon_boundary_wkb=self.taos_shape.wkb, spatial_resolution_m=120) self.assertIsNotNone(nda) self.assertEqual((902, 648, 3), nda.shape) band_numbers = [Band.RED, Band.BLUE, Band.GREEN] nda = landsat.fetch_imagery_array(band_numbers, scaleParams, spatial_resolution_m=120) self.assertIsNotNone(nda) self.assertNotEqual((902, 648, 3), nda.shape) def test_two_bands(self): # specify the bands that approximate real color landsat = Landsat(self.metadata_set) band_numbers = [Band.RED, Band.BLUE] scaleParams = [[0.0, 40000.0], [0.0, 40000.0]] nda = landsat.fetch_imagery_array( band_numbers, scaleParams, polygon_boundary_wkb=self.taos_shape.wkb, spatial_resolution_m=120) self.assertIsNotNone(nda) self.assertEqual((2, 902, 648), nda.shape) def test_one_band(self): # specify the bands that approximate real color landsat = Landsat(self.metadata_set) band_numbers = [Band.RED] scaleParams = [[0.0, 40000.0]] nda = landsat.fetch_imagery_array( band_numbers, scaleParams, polygon_boundary_wkb=self.taos_shape.wkb, spatial_resolution_m=120) self.assertIsNotNone(nda) self.assertEqual((902, 648), nda.shape)
class TestGCPLandsat(unittest.TestCase): base_mount_path = '/imagery' metadata_service = None metadata_set = [] r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json") taos_geom = r.json() taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry']) def setUp(self): d_start = date(2017, 3, 12) # 2017-03-12 d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive self.metadata_service = MetadataService() landsat_filters = LandsatQueryFilters() landsat_filters.collection_number.set_value("PRE") landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*self.taos_shape.bounds) rows = self.metadata_service.search( SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filters) rows = list(rows) # mounted directory in docker container base_mount_path = '/imagery' for row in rows: self.metadata_set.append(row) def test_gdal_info(self): d_start = date(2015, 6, 24) d_end = date(2016, 6, 24) bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799) landsat_filters = LandsatQueryFilters() landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = self.metadata_service.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filters, limit=1) rows = list(rows) metadata = rows[0] storage = Storage(metadata.bucket_name) b_mounted = storage.mount_sub_folder(metadata, "generic") self.assertTrue(b_mounted) b_deleted = storage.unmount_sub_folder(metadata, "generic") self.assertTrue(b_deleted) def test_landsat5_vrt(self): # 5th Place: Lake Eyre Landsat 5 Acquired August 5, 2006 wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \ "138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \ "2469482421875 -27.57843813308233))" polygon = loads(wkt) # sql_filters = ['cloud_cover=0'] d_start = date(2006, 8, 4) d_end = date(2006, 8, 5) bounding_box = polygon.bounds landsat_filters = LandsatQueryFilters() landsat_filters.wrs_row.set_value(79) landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = self.metadata_service.search( SpacecraftID.LANDSAT_5, data_filters=landsat_filters) rows = list(rows) self.assertEqual(len(rows), 1) # data structure that contains all fields from Google's Landsat BigQuery Database metadata = rows[0] # GDAL helper functions for generating VRT landsat = Landsat(metadata) vrt = landsat.get_vrt([3, 2, 1]) with open('testlandsat5.vrt', 'r') as myfile: data = myfile.read() expected = etree.XML(data) actual = etree.XML(vrt) result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10}) self.assertTrue(result, message) def test_australia(self): # 5th Place: Lake Eyre Landsat 5 Acquired August 5, 2006 wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \ "138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \ "2469482421875 -27.57843813308233))" polygon = loads(wkt) # sql_filters = ['cloud_cover=0'] d_start = date(2006, 8, 4) d_end = date(2006, 8, 7) bounding_box = polygon.bounds landsat_filters = LandsatQueryFilters() landsat_filters.wrs_row.set_value(79) landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = self.metadata_service.search( SpacecraftID.LANDSAT_5, data_filters=landsat_filters) rows = list(rows) self.assertEqual(len(rows), 1) metadata = rows[0] landsat = Landsat(metadata) # get a numpy.ndarray from bands for specified imagery band_numbers = [3, 2, 1] scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]] # nda = landsat.__get_ndarray(band_numbers, metadata, scale_params) nda = landsat.fetch_imagery_array(band_numbers, scale_params) self.assertEqual((3581, 4046, 3), nda.shape) def test_unmount_destructor(self): wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \ "138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \ "2469482421875 -27.57843813308233))" polygon = loads(wkt) # sql_filters = ['cloud_cover=0'] d_start = date(2006, 8, 4) d_end = date(2006, 8, 7) bounding_box = polygon.bounds landsat_filters = LandsatQueryFilters() landsat_filters.wrs_row.set_value(79) landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = self.metadata_service.search( SpacecraftID.LANDSAT_5, data_filters=landsat_filters) rows = list(rows) metadata = rows[0] landsat = Landsat(metadata) vrt = landsat.get_vrt([4]) # storage = Storage("gcp-public-data-landsat") # del landsat # self.assertFalse(storage.is_mounted(metadata)) def test_unmount_destructor_conflict(self): wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \ "138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \ "2469482421875 -27.57843813308233))" polygon = loads(wkt) # sql_filters = ['cloud_cover=0'] d_start = date(2006, 8, 4) d_end = date(2006, 8, 7) bounding_box = polygon.bounds # sql_filters = ['wrs_row=79'] landsat_filters = LandsatQueryFilters() landsat_filters.wrs_row.set_value(79) landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = self.metadata_service.search( SpacecraftID.LANDSAT_5, data_filters=landsat_filters) rows = list(rows) metadata = rows[0] landsat = Landsat(metadata) vrt = landsat.get_vrt([4]) storage = Storage("gcp-public-data-landsat") landsat_2 = Landsat(metadata) vrt = landsat_2.get_vrt([4]) del landsat self.assertTrue(storage.is_mounted(metadata)) def test_vrt(self): d_start = date(2015, 6, 24) d_end = date(2016, 6, 24) bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799) # sql_filters = ['scene_id="LC80400312016103LGN00"'] landsat_filters = LandsatQueryFilters() landsat_filters.scene_id.set_value("LC80400312016103LGN00") landsat_filters.acquired.set_range(d_start, True, d_end, True) landsat_filters.aoi.set_bounds(*bounding_box) rows = self.metadata_service.search(SpacecraftID.LANDSAT_8, limit=1, data_filters=landsat_filters) rows = list(rows) metadata = rows[0] landsat = Landsat(metadata) vrt = landsat.get_vrt([4, 3, 2]) with open('test_1.vrt', 'r') as myfile: data = myfile.read() expected = etree.XML(data) actual = etree.XML(vrt) result, message = xml_compare(expected, actual) self.assertTrue(result, message) dataset = gdal.Open(vrt) self.assertIsNotNone(dataset) ds_band_1 = dataset.GetRasterBand(1) self.assertIsNotNone(ds_band_1) self.assertEqual(ds_band_1.XSize, 7631) ds_band_2 = dataset.GetRasterBand(2) self.assertIsNotNone(ds_band_2) self.assertEqual(ds_band_2.YSize, 7771) ds_band_3 = dataset.GetRasterBand(3) self.assertIsNotNone(ds_band_3) self.assertEqual(ds_band_3.YSize, 7771)