def test_cloud_cover(self):
metadata_service = MetadataService()
landsat_filters = LandsatQueryFilters()
landsat_filters.cloud_cover.set_value(0)
# landsat_filters.scene_id.set_value("LC80390332016208LGN00")
# sql_filters = ['cloud_cover=0']
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999,
44.84029065139799)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 10)
for row in rows:
self.assertEqual(row.spacecraft_id.name,
SpacecraftID.LANDSAT_8.name)
d_actual = datetime.datetime.strptime(row.date_acquired,
'%Y-%m-%d').date()
self.assertLessEqual(d_actual, d_end)
self.assertGreaterEqual(d_actual, d_start)
test_box = row.bounds
self.assertTrue(
(bounding_box[0] < test_box[2] < bounding_box[2])
or (bounding_box[0] < test_box[0] < bounding_box[2]))
self.assertTrue(
(bounding_box[1] < test_box[3] < bounding_box[3])
or (bounding_box[1] < test_box[1] < bounding_box[3]))
def setUp(self):
metadata_service = MetadataService()
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80400312016103LGN00")
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=1,
data_filters=landsat_filters)
rows = list(rows)
self.m_row_data = rows[0]
wkt_iowa = "POLYGON((-93.76075744628906 42.32707774458643,-93.47854614257812 42.32707774458643," \
"-93.47854614257812 42.12674735753131,-93.76075744628906 42.12674735753131," \
"-93.76075744628906 42.32707774458643))"
self.iowa_polygon = loads(wkt_iowa)
gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', "YES")
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_exclude_value("PRE")
landsat_filters.acquired.set_range(start=d_start, end=d_end)
landsat_filters.aoi.set_bounds(*self.taos_shape.bounds)
landsat_filters.data_type.set_exclude_value('L1GT')
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8,
limit=10,
data_filters=landsat_filters)
for row in rows:
self.metadata_set.append(row)
def test_end_date(self):
r = requests.get(
"https://raw.githubusercontent.com/johan/world.geo.json/master/countries/BEL.geo.json"
)
area_geom = r.json()
area_shape = shapely.geometry.shape(
area_geom['features'][0]['geometry'])
# gs://gcp-public-data-landsat/LC08/PRE/044/034/LC80440342016259LGN00/
metadata_service = MetadataService()
d = date(2016, 6, 24)
landsat_filter = LandsatQueryFilters()
landsat_filter.acquired.set_range(end=d, end_inclusive=True)
landsat_filter.acquired.sort_by(epl_imagery_pb2.DESCENDING)
landsat_filter.aoi.set_bounds(*area_shape.bounds)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filter)
rows = list(rows)
self.assertEqual(len(rows), 10)
d_previous = d
for row in rows:
self.assertEqual(row.spacecraft_id.name,
SpacecraftID.LANDSAT_8.name)
d_actual = datetime.datetime.strptime(row.date_acquired,
'%Y-%m-%d').date()
self.assertLessEqual(d_actual, d_previous)
d_previous = d_actual
def test_polygon_boundary(self):
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive
r = requests.get(
"https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json"
)
taos_geom = r.json()
taos_shape = shapely.geometry.shape(
taos_geom['features'][0]['geometry'])
metadata_service = MetadataService()
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_geometry(taos_shape.wkb)
# landsat_filters.geometry_bag.geometry_binaries.append(taos_shape.wkb)
metadata_rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=10,
data_filters=landsat_filters)
# mounted directory in docker container
metadata_set = []
for row in metadata_rows:
metadata_set.append(row)
self.assertEqual(len(metadata_set), 2)
def test_start_date(self):
# gs://gcp-public-data-landsat/LC08/PRE/044/034/LC80440342016259LGN00/
metadata_service = MetadataService()
d = date(2016, 6, 24)
landsat_filters = LandsatQueryFilters()
landsat_filters.acquired.set_range(d, True)
landsat_filters.wrs_path.set_value(125)
landsat_filters.wrs_row.set_value(49)
landsat_filters.acquired.sort_by(epl_imagery_pb2.ASCENDING)
landsat_filters.acquired.set_range(end=d, end_inclusive=True)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 10)
d_previous = datetime.datetime.strptime("1945-01-01",
'%Y-%m-%d').date()
for row in rows:
self.assertEqual(row.spacecraft_id.name,
SpacecraftID.LANDSAT_8.name)
d_actual = datetime.datetime.strptime(row.date_acquired,
'%Y-%m-%d').date()
# test Order by
self.assertGreaterEqual(d_actual, d_previous)
d_previous = d_actual
def test_aws_file_path(self):
# PRE s3://landsat-pds/L8/139/045/LC81390452014295LGN00/
# non-PRE s3://landsat-pds/c1/L8/139/045/LC08_L1TP_139045_20170304_20170316_01_T1/
metadataservice = MetadataService()
start_date = datetime.datetime.strptime('14295', '%y%j').date()
landsat_filters = LandsatQueryFilters()
landsat_filters.wrs_path.set_value(139)
landsat_filters.wrs_row.set_value(45)
landsat_filters.acquired.set_range(start_date, True, start_date, True)
landsat_filters.collection_number.set_value("PRE")
rows = metadataservice.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
# turn gernator into list
metadata_set = list(rows)
self.assertEqual(len(metadata_set), 1)
metadata = metadata_set[0]
self.assertEqual(metadata.get_aws_file_path(), "/imagery/L8/139/045/LC81390452014295LGN00")
landsat_filters = LandsatQueryFilters()
landsat_filters.wrs_path.set_value(139)
landsat_filters.wrs_row.set_value(45)
# landsat_filters.collection_number.set_exclude_value("PRE")
landsat_filters.acquired.set_range(start=date(2017, 3, 4), end=date(2017, 3, 4))
rows = metadataservice.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
metadata_set = list(rows)
self.assertEqual(len(metadata_set), 2)
metadata = metadata_set[0]
self.assertEqual(metadata.get_aws_file_path(),
"/imagery/c1/L8/139/045/LC08_L1TP_139045_20170304_20170316_01_T1")
metadata = metadata_set[1]
self.assertEqual(metadata.get_aws_file_path(),
"/imagery/L8/139/045/LC81390452017063LGN00")
def test_aws_without_google(self):
metadata_service = MetadataService()
# c1/L8/083/015/LC08_L1TP_083015_20171106_20171107_01_RT/
# c1/L8/083/111/LC08_L1GT_083111_20171106_20171107_01_RT
sqs_message = {
"Type": "Notification",
"MessageId": "27f57c3d-9d2e-5fa3-8f83-2e41a3aa5634",
"TopicArn": "arn:aws:sns:us-west-2:274514004127:NewSceneHTML",
"Subject": "Amazon S3 Notification",
"Message": "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"us-west-2\",\"eventTime\":\"2017-11-07T23:05:40.162Z\",\"eventName\":\"ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"AWS:AIDAILHHXPNIKSGVUGOZK\"},\"requestParameters\":{\"sourceIPAddress\":\"35.193.238.175\"},\"responseElements\":{\"x-amz-request-id\":\"F96A6CC9816FC5EF\",\"x-amz-id-2\":\"yehs3XxTY8utc9kgKfNbMe1wdtV7F0wEMUXUQtIu7zMRtGvboxahzwncrmG046yI327j5IRh8nE=\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"C1-NewHTML\",\"bucket\":{\"name\":\"landsat-pds\",\"ownerIdentity\":{\"principalId\":\"A3LZTVCZQ87CNW\"},\"arn\":\"arn:aws:s3:::landsat-pds\"},\"object\":{\"key\":\"c1/L8/115/062/LC08_L1TP_115062_20171107_20171107_01_RT/index.html\",\"size\":5391,\"eTag\":\"0f06667fca1f707894bf579bd667e221\",\"sequencer\":\"005A023C441A8F8403\"}}}]}",
"Timestamp": "2017-11-07T23:05:40.219Z",
"SignatureVersion": "1",
"Signature": "bqrW1x6CgJntCz6f0F5uncyPZR+6ZM/tZ3OrRZDiudBv5DAtMyYR9n6KQ0aT+iYP5INfpL2GuIm8Uqco8ZHzg5AqEhHtNkpzGBQpQHvlF3t0ut9K27YNwJ6ZmnS14BgsLWyXIthVRjvHf1Hhx3ZInPMJrzTcKCOhOmBcM9zOpfWrHfnynuifpN3FaldDz6VY2d9QM0Rn8Fo8XZ4F+j01eAJVlydnRbSBbLewleuvhPQh6EG5r2EeekeniOIETrodS7o43ZClFr8OSgRE7BvpecVnnUEXBUIDDtRAPnIxo3Io0AmfPRI8xRfeKNhBIhPq3W3clm7Dxkp3N96OKoVUBw==",
"SigningCertURL": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-433026a4050d206028891664da859041.pem",
"UnsubscribeURL": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-west-2:274514004127:NewSceneHTML:7997d757-d1c6-4064-8935-34111968c8cc"
}
d = datetime.datetime.strptime(sqs_message['Timestamp'], "%Y-%m-%dT%H:%M:%S.%fZ")
rows = metadata_service.search_aws('/imagery', wrs_path=115, wrs_row=62, collection_date=d)
self.assertGreater(len(rows), 0)
metadata = rows[0]
self.assertIsNotNone(metadata)
landsat = Landsat(metadata)
nda = landsat.fetch_imagery_array([4, 3, 2], [[0, 40000], [0, 40000], [0, 40000]], spatial_resolution_m=240)
self.assertIsNotNone(nda)
def test_polygon_wkb_metadata(self):
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive
self.metadata_service = MetadataService()
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_geometry(self.taos_shape.wkb)
# landsat_filters.geometry_bag.geometry_binaries.append(self.taos_shape.wkb)
metadata_rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filters)
metadata_set = []
for row in metadata_rows:
metadata_set.append(row)
landsat = Landsat(metadata_set)
band_numbers = [Band.NIR, Band.SWIR1, Band.SWIR2]
scaleParams = [[0.0, 40000.0], [0.0, 40000.0], [0.0, 40000.0]]
nda = landsat.fetch_imagery_array(
band_numbers,
scaleParams,
polygon_boundary_wkb=self.taos_shape.wkb)
self.assertIsNotNone(nda)
self.assertEqual((1804, 1295, 3), nda.shape)
def test_metatdata_file_list(self):
wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \
"138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \
"2469482421875 -27.57843813308233))"
polygon = loads(wkt)
metadata_service = MetadataService()
# sql_filters = ['cloud_cover=0']
d_start = date(2006, 8, 4)
d_end = date(2006, 8, 5)
bounding_box = polygon.bounds
# sql_filters = ['wrs_row=79']
landsat_filters = LandsatQueryFilters()
# landsat_filters.wrs_path.set_value(125)
landsat_filters.wrs_row.set_value(79)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = metadata_service.search(
SpacecraftID.LANDSAT_5,
data_filters=landsat_filters)
rows = list(rows)
metadata = rows[0]
self.assertEqual(len(metadata.get_file_list()), 0)
def test_scene_id(self):
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80390332016208LGN00")
metadata_service = MetadataService()
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 1)
def test_where_start(self):
# sql_filters = ['scene_id="LC80270312016188LGN00"']
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80270312016188LGN00")
metadata_service = MetadataService()
metadata_rows = metadata_service.search(
SpacecraftID.UNKNOWN_SPACECRAFT, data_filters=landsat_filters)
metadata_set = list(metadata_rows)
self.assertEqual(1, len(metadata_set))
def test_delete_storage(self):
metadata_service = MetadataService()
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters,
limit=1)
rows = list(rows)
metadata = rows[0]
def test_alaskan_aleutians(self):
# wkt = "MULTIPOLYGON (((40 40, 20 45, 45 30, 40 40)), ((-172 53, 175 53, 175 48, -172 48, -172 53)), ((20 35, 45 20, 30 5, 10 10, 10 30, 20 35), (30 20, 20 25, 20 15, 30 20)))"
wkt = "POLYGON((-172 53, 175 53, 175 48, -172 48, -172 53))"
islands_shape = loads(wkt)
self.assertFalse(islands_shape.exterior.is_ccw)
query_filter = LandsatQueryFilters()
query_filter.aoi.set_geometry(islands_shape.wkb)
# query_filter.geometry_bag.geometry_binaries.append(islands_shape.wkb)
metadata_service = MetadataService()
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=10,
data_filters=query_filter)
stuff = list(rows)
self.assertEqual(10, len(stuff))
def test_no_bounding_box(self):
d_start = date(2003, 4, 4)
d_end = date(2003, 4, 7)
landsat_filters = LandsatQueryFilters()
landsat_filters.wrs_path.set_value(125)
landsat_filters.wrs_row.set_value(49)
# sql_filters = ['wrs_row=49', 'wrs_path=125']
metadata_service = MetadataService()
landsat_filters.acquired.set_range(d_start, True, d_end, True)
rows = metadata_service.search(
satellite_id=None,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 3)
def test_salt_lake_city(self):
utah_wkt = 'POLYGON((-114.049883347847 38.677365, -114.049749 38.72920999999999, -114.049168 38.749951, -114.049465 38.874949, -114.048521 38.876197, -114.048054 38.878693, -114.049104 39.005509, -114.047079 39.49994299999999, -114.047727981839 39.5427408023268, -113.815766 39.54409, -113.815743 39.552644, -112.923426 39.552539, -112.803046 39.552648, -112.629539 39.5524, -112.462419 39.552451, -112.462423 39.553704, -112.354467 39.553684, -112.235958 39.553625, -112.212045 39.553987, -112.212029 39.54773, -112.207353 39.54769599999999, -112.207346 39.54408, -112.202672 39.544048, -112.202666 39.540434, -112.193292 39.540451, -112.193242 39.526023, -112.211973 39.526042, -112.211961 39.511579, -112.207285 39.51156599999999, -112.2073 39.504334, -112.202633 39.504323, -112.202626 39.489858, -112.193277 39.48983399999999, -112.193596 39.37309, -112.188907 39.373089, -112.188922 39.329392, -112.137293 39.329397, -112.10962 39.329522, -112.072108 39.329695, -112.072218 39.314923, -112.053423 39.31494199999999, -112.053421 39.314483, -112.016003 39.314561, -112.01452 39.132617, -112.014083 39.045518, -112.014017 39.024282, -112.019034 39.024207, -112.018906 38.995902, -112.056387 38.99568, -112.056402 38.988368, -112.065661 38.98843, -112.06563 38.985359, -112.065591 38.981689, -112.065162 38.959302, -112.133862 38.959302, -112.134031 38.935453, -112.151928 38.935473, -112.150513 38.92005899999999, -112.15076 38.906509, -112.169351 38.906132, -112.169461 38.892104, -112.171554 38.878616, -112.18847 38.879447, -112.188541 38.864787, -112.199098 38.864982, -112.206093 38.865112, -112.224461 38.86480299999999, -112.224553 38.85556, -112.224637 38.837514, -112.228438 38.837508, -112.228381 38.763987, -112.219118 38.763987, -112.219116 38.751251, -112.21808 38.748987, -112.218147 38.740198, -112.218356 38.72736099999999, -112.23664 38.727104, -112.236685 38.723266, -112.236722 38.719859, -112.241362 38.719785, -112.24147 38.71252399999999, -112.255359 38.712306, -112.255438 38.705047, -112.273297 38.704999, -112.273255 38.701353, -112.28235 38.701527, -112.28226 38.686874, -112.291316 38.686961, -112.291329 38.683298, -112.30994 38.68340999999999, -112.309953 38.67629, -112.347179 38.676259, -112.356637 38.67674, -112.356539 38.683855, -112.365964 38.684218, -112.384245 38.68331, -112.401176 38.680972, -112.4198 38.681388, -112.447517 38.68127, -112.447667 38.67392299999999, -112.466001 38.673917, -112.466156 38.659267, -112.475341 38.659257, -112.475185 38.644675, -112.484444 38.644647, -112.484788 38.600632, -112.503074 38.600568, -112.503148 38.571377, -112.51239 38.571358, -112.515394 38.572845, -112.603479 38.572733, -112.752497 38.572681, -113.047215 38.572603, -113.115956 38.572663, -113.11686 38.572612, -113.189536 38.57273199999999, -113.191158 38.57261099999999, -114.05015385888 38.5729744583009, -114.049883347847 38.677365))'
utah_shape = shapely.wkt.loads(utah_wkt)
landsat_qf = LandsatQueryFilters()
# cloud cover less than 30%
landsat_qf.cloud_cover.set_range(end=30)
landsat_qf.aoi.set_geometry(utah_shape.wkb)
# sort by date, with most recent first
landsat_qf.acquired.sort_by(epl_imagery_pb2.DESCENDING)
metadata_servce = MetadataService()
rows = metadata_servce.search_layer_group(data_filters=landsat_qf,
satellite_id=SpacecraftID.LANDSAT_8)
self.assertIsNotNone(rows)
metadata_set = list(rows)
self.assertGreaterEqual(len(metadata_set), 1)
def test_split_wrs(self):
wkt = "POLYGON((-172 53, 175 53, 175 48, -172 48, -172 53))"
islands_shape = loads(wkt)
metadata_service = MetadataService()
wrs_set = metadata_service.get_wrs([islands_shape.wkb],
search_area_unioned=islands_shape)
sorted_overlaps = metadata_service.sorted_wrs_overlaps(
wrs_set=wrs_set, search_area=islands_shape)
self.assertIsNotNone(wrs_set)
self.assertIsNotNone(sorted_overlaps)
self.assertGreater(len(wrs_set), 0)
self.assertGreater(len(sorted_overlaps), 0)
prev_area = 0
for wrs_overlap in sorted_overlaps:
self.assertLessEqual(prev_area, wrs_overlap[2])
prev_area = wrs_overlap[2]
def test_one_day(self):
# gs://gcp-public-data-landsat/LC08/PRE/044/034/LC80440342016259LGN00/
metadata_service = MetadataService()
d = date(2016, 6, 24)
landsat_filter = LandsatQueryFilters()
landsat_filter.acquired.set_value(d)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filter)
rows = list(rows)
self.assertEqual(len(rows), 10)
for row in rows:
self.assertEqual(row.spacecraft_id.name,
SpacecraftID.LANDSAT_8.name)
d_actual = datetime.datetime.strptime(row.date_acquired,
'%Y-%m-%d').date()
self.assertEqual(d_actual, d)
def test_google_aws_mismatch(self):
metadata_service = MetadataService()
d_start = date(2017, 6, 24)
d_end = date(2017, 9, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_exclude_value("PRE")
landsat_filters.acquired.set_range(start=d_start, end=d_end)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=2,
data_filters=landsat_filters)
# generator to list
rows = list(rows)
metadata = rows[0]
# self.assertEqual('LC08_L1GT_135215_20170916_20170916_01_RT', metadata.product_id)
self.assertEqual('RT', metadata.collection_category)
def setUp(self):
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive
self.metadata_service = MetadataService()
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*self.taos_shape.bounds)
metadata_rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filters)
# mounted directory in docker container
base_mount_path = '/imagery'
for row in metadata_rows:
self.metadata_set.append(row)
def test_search_group_geometry(self):
import shapely.wkb
r = requests.get(
"https://raw.githubusercontent.com/johan/world.geo.json/master/countries/BEL.geo.json"
)
area_geom = r.json()
area_shape = shapely.geometry.shape(
area_geom['features'][0]['geometry'])
d_start = date(2017, 1, 1) # 2017-03-12
d_end = date(2017, 5, 19) # epl api is inclusive
belgium_filter = LandsatQueryFilters()
# PRE is a collection type that specifies certain QA standards
# belgium_filter.collection_number.set_value("PRE")
belgium_filter.cloud_cover.set_range(end=15, end_inclusive=False)
belgium_filter.acquired.set_range(start=d_start, end=d_end)
belgium_filter.aoi.set_geometry(area_shape.wkb)
# belgium_filter.aoi.set_bounds(*area_shape.bounds)
# search the satellite metadata for images of Belgium withing the given date range
metadata_service = MetadataService()
metadata_gen = metadata_service.search_layer_group(
data_filters=belgium_filter, satellite_id=SpacecraftID.LANDSAT_8)
unioned_beast = shapely.geometry.Polygon()
for metadata in metadata_gen:
wrs_polygon = metadata.get_wrs_polygon()
wrs_shape = shapely.wkb.loads(wrs_polygon)
unioned_beast = unioned_beast.union(wrs_shape)
self.assertTrue(unioned_beast.contains(area_shape))
belgium_filter.aoi.sort_by(epl_imagery_pb2.DESCENDING)
metadata_gen = metadata_service.search_layer_group(
data_filters=belgium_filter, satellite_id=SpacecraftID.LANDSAT_8)
unioned_beast = shapely.geometry.Polygon()
for metadata in metadata_gen:
wrs_polygon = metadata.get_wrs_polygon()
wrs_shape = shapely.wkb.loads(wrs_polygon)
unioned_beast = unioned_beast.union(wrs_shape)
self.assertTrue(unioned_beast.contains(area_shape))
def test_get_file(self):
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
metadata_service = MetadataService()
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters,
limit=1)
rows = list(rows)
metadata = rows[0]
landsat = Landsat(metadata)
self.assertIsNotNone(landsat)
vrt = landsat.get_vrt([4, 3, 2])
self.assertIsNotNone(vrt)
dataset = landsat.get_dataset([4, 3, 2], DataType.UINT16)
self.assertIsNotNone(dataset)
def test_metadata_extent(self):
r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json")
taos_geom = r.json()
print(taos_geom)
taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry'])
metadata_service = MetadataService()
# sql_filters = ['scene_id="LC80330342017072LGN00"', 'collection_number="PRE"']
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80330342017072LGN00")
landsat_filters.collection_number.set_value("PRE")
rows = metadata_service.search(
SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 1)
metadata = rows[0]
# GDAL helper functions for generating VRT
landsat = Landsat(metadata)
# get a numpy.ndarray from bands for specified imagery
band_numbers = [Band.RED, Band.GREEN, Band.BLUE]
scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]]
vrt = landsat.get_vrt(band_numbers, envelope_boundary=taos_shape.bounds)
with open('clipped_LC80330342017072LGN00.vrt', 'r') as myfile:
data = myfile.read()
expected = etree.XML(data)
actual = etree.XML(vrt)
result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10, "xOff": 1e-10, "yOff": 1e-10})
self.assertTrue(result, message)
dataset = gdal.Open(vrt)
geo_transform = dataset.GetGeoTransform()
# self.assertEqual(geo_transform, raster_metadata.get_geotransform(taos_shape.bounds))
# self.assertNotEqual(geo_transform, raster_metadata.get_geotransform())
"""
def test_storage_create(self):
metadata_service = MetadataService()
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters,
limit=1)
rows = list(rows)
metadata = rows[0]
storage = Storage(metadata.bucket_name)
metadata = rows[0]
self.assertTrue(storage.mount_sub_folder(metadata, "generic"))
self.assertTrue(storage.unmount_sub_folder(metadata, "generic"))
def test_bounds(self):
metadata_service = MetadataService()
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80330342017072LGN00")
landsat_filters.collection_number.set_value("PRE")
rows = metadata_service.search(
SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 1)
metadata = rows[0]
bands = [Band.RED, Band.BLUE, Band.GREEN]
band_map = BandMap(SpacecraftID.LANDSAT_8)
raster_metadata = RasterMetadata()
storage = Storage()
storage.mount_sub_folder(metadata)
for band in bands:
band_number = band_map.get_number(band)
raster_metadata.add_metadata(band_number, metadata)
boundary = raster_metadata.bounds
self.assertIsNotNone(boundary)
r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json")
taos_geom = r.json()
taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry'])
clipped_raster = raster_metadata.calculate_clipped(taos_shape.bounds, pyproj.Proj(init='epsg:4326'))
self.assertIsNotNone(clipped_raster.bounds)
big_box = shapely.geometry.box(*boundary)
small_box = shapely.geometry.box(*clipped_raster.bounds)
self.assertTrue(big_box.contains(small_box))
def test_colorado(self):
r = requests.get(
"https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/CO/Boulder.geo.json"
)
boulder_geom = r.json()
boulder_shape = shapely.geometry.shape(
boulder_geom['features'][0]['geometry'])
d_start = date(2017, 3, 1) # 2017-03-12
d_end = date(2017, 3, 19) # epl api is inclusive
# PRE is a collection type that specifies certain QA standards
landsat_filter = LandsatQueryFilters()
landsat_filter.collection_number.set_value("PRE")
landsat_filter.acquired.set_range(d_start, True, d_end, True)
landsat_filter.aoi.set_bounds(*boulder_shape.bounds)
# search the satellite metadata for images of Taos withing the given date range
metadata_service = MetadataService()
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=10,
data_filters=landsat_filter)
self.assertTrue(2, len(list(rows)))
def test_belgium(self):
r = requests.get(
"https://raw.githubusercontent.com/johan/world.geo.json/master/countries/BEL.geo.json"
)
area_geom = r.json()
area_shape = shapely.geometry.shape(
area_geom['features'][0]['geometry'])
d_start = date(2017, 1, 1) # 2017-03-12
d_end = date(2017, 5, 19) # epl api is inclusive
belgium_filter = LandsatQueryFilters()
# PRE is a collection type that specifies certain QA standards
belgium_filter.collection_number.set_value("PRE")
belgium_filter.cloud_cover.set_range(end=15, end_inclusive=False)
belgium_filter.acquired.set_range(start=d_start, end=d_end)
belgium_filter.aoi.set_bounds(*area_shape.bounds)
# search the satellite metadata for images of Belgium withing the given date range
metadata_service = MetadataService()
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=20,
data_filters=belgium_filter)
def test_all_sat_data(self):
metadata_service = MetadataService()
landsat_filters = LandsatQueryFilters()
landsat_filters.cloud_cover.set_value(0)
d_start = date(2004, 6, 24)
d_end = date(2008, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -114.31054687499999, 35.84029065139799)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = metadata_service.search(
data_filters=landsat_filters)
rows = list(rows)
first_item = rows[0]
self.assertEqual(len(rows), 10)
rows = metadata_service.search(
satellite_id=SpacecraftID.UNKNOWN_SPACECRAFT,
data_filters=landsat_filters)
rows = list(rows)
other_item = rows[0]
self.assertEqual(len(rows), 10)
self.assertEqual(first_item.scene_id, other_item.scene_id)
def setUp(self):
self.metadata_service = MetadataService()
class TestAWSPixelFunctions(unittest.TestCase):
m_row_data = None
base_mount_path = '/imagery'
metadata_service = MetadataService()
iowa_polygon = None
metadata_set = []
r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json")
taos_geom = r.json()
taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry'])
def setUp(self):
metadata_service = MetadataService()
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80400312016103LGN00")
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=1,
data_filters=landsat_filters)
rows = list(rows)
self.m_row_data = rows[0]
wkt_iowa = "POLYGON((-93.76075744628906 42.32707774458643,-93.47854614257812 42.32707774458643," \
"-93.47854614257812 42.12674735753131,-93.76075744628906 42.12674735753131," \
"-93.76075744628906 42.32707774458643))"
self.iowa_polygon = loads(wkt_iowa)
gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', "YES")
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_exclude_value("PRE")
landsat_filters.acquired.set_range(start=d_start, end=d_end)
landsat_filters.aoi.set_bounds(*self.taos_shape.bounds)
landsat_filters.data_type.set_exclude_value('L1GT')
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8,
limit=10,
data_filters=landsat_filters)
for row in rows:
self.metadata_set.append(row)
def test_pixel_1(self):
metadata = self.m_row_data
landsat = Landsat(metadata) # , gsurl[2])
code = """import numpy as np
def multiply_rounded(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize,
raster_ysize, buf_radius, gt, **kwargs):
factor = float(kwargs['factor'])
out_ar[:] = np.round_(np.clip(in_ar[0] * factor,0,255))"""
function_arguments = {"factor": "1.5"}
pixel_function_details = FunctionDetails(name="multiply_rounded", band_definitions=[2],
data_type=DataType.FLOAT32, code=code,
arguments=function_arguments)
vrt = landsat.get_vrt([pixel_function_details, 3, 2])
with open('pixel_1_aws.vrt', 'r') as myfile:
data = myfile.read()
expected = etree.XML(data)
actual = etree.XML(vrt)
result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10})
self.assertTrue(result, message)
def test_missing_data(self):
start = date(2017, 4, 15)
end = date(2017, 5, 15)
r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/BEL.geo.json")
area_geom = r.json()
area_shape = shapely.geometry.shape(area_geom['features'][0]['geometry'])
landsat_filter = LandsatQueryFilters()
landsat_filter.acquired.set_range(start=start, end=end)
landsat_filter.aoi.set_bounds(*area_shape.bounds)
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8,
limit=40,
data_filters=landsat_filter)
metadataset= []
for row in rows:
metadataset.append(row)
self.assertEqual(40, len(metadataset))
def test_json_txt_mtl(self):
failed_2 = "/imagery/c1/L8/089/078/LC08_L1TP_089078_20180612_20180613_01_RT"
metadata2 = Metadata(failed_2)
self.assertIsNotNone(metadata2)
self.assertEqual(89, metadata2.wrs_path)
self.assertEqual(78, metadata2.wrs_row)
date(2018, 6, 12).isoformat()
self.assertGreaterEqual(date(2018, 6, 12).isoformat(), metadata2.date_acquired)
def test_missing_mtl_json(self):
landsat_filter = LandsatQueryFilters()
landsat_filter.product_id.set_value("LC08_L1TP_089078_20180612_20180613_01_RT")
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8,
limit=1,
data_filters=landsat_filter)
metadata_set = list(rows)
self.assertEqual(1, len(metadata_set))
def test_salt_lake_city(self):
utah_wkt = 'POLYGON((-114.049883347847 38.677365, -114.049749 38.72920999999999, -114.049168 38.749951, -114.049465 38.874949, -114.048521 38.876197, -114.048054 38.878693, -114.049104 39.005509, -114.047079 39.49994299999999, -114.047727981839 39.5427408023268, -113.815766 39.54409, -113.815743 39.552644, -112.923426 39.552539, -112.803046 39.552648, -112.629539 39.5524, -112.462419 39.552451, -112.462423 39.553704, -112.354467 39.553684, -112.235958 39.553625, -112.212045 39.553987, -112.212029 39.54773, -112.207353 39.54769599999999, -112.207346 39.54408, -112.202672 39.544048, -112.202666 39.540434, -112.193292 39.540451, -112.193242 39.526023, -112.211973 39.526042, -112.211961 39.511579, -112.207285 39.51156599999999, -112.2073 39.504334, -112.202633 39.504323, -112.202626 39.489858, -112.193277 39.48983399999999, -112.193596 39.37309, -112.188907 39.373089, -112.188922 39.329392, -112.137293 39.329397, -112.10962 39.329522, -112.072108 39.329695, -112.072218 39.314923, -112.053423 39.31494199999999, -112.053421 39.314483, -112.016003 39.314561, -112.01452 39.132617, -112.014083 39.045518, -112.014017 39.024282, -112.019034 39.024207, -112.018906 38.995902, -112.056387 38.99568, -112.056402 38.988368, -112.065661 38.98843, -112.06563 38.985359, -112.065591 38.981689, -112.065162 38.959302, -112.133862 38.959302, -112.134031 38.935453, -112.151928 38.935473, -112.150513 38.92005899999999, -112.15076 38.906509, -112.169351 38.906132, -112.169461 38.892104, -112.171554 38.878616, -112.18847 38.879447, -112.188541 38.864787, -112.199098 38.864982, -112.206093 38.865112, -112.224461 38.86480299999999, -112.224553 38.85556, -112.224637 38.837514, -112.228438 38.837508, -112.228381 38.763987, -112.219118 38.763987, -112.219116 38.751251, -112.21808 38.748987, -112.218147 38.740198, -112.218356 38.72736099999999, -112.23664 38.727104, -112.236685 38.723266, -112.236722 38.719859, -112.241362 38.719785, -112.24147 38.71252399999999, -112.255359 38.712306, -112.255438 38.705047, -112.273297 38.704999, -112.273255 38.701353, -112.28235 38.701527, -112.28226 38.686874, -112.291316 38.686961, -112.291329 38.683298, -112.30994 38.68340999999999, -112.309953 38.67629, -112.347179 38.676259, -112.356637 38.67674, -112.356539 38.683855, -112.365964 38.684218, -112.384245 38.68331, -112.401176 38.680972, -112.4198 38.681388, -112.447517 38.68127, -112.447667 38.67392299999999, -112.466001 38.673917, -112.466156 38.659267, -112.475341 38.659257, -112.475185 38.644675, -112.484444 38.644647, -112.484788 38.600632, -112.503074 38.600568, -112.503148 38.571377, -112.51239 38.571358, -112.515394 38.572845, -112.603479 38.572733, -112.752497 38.572681, -113.047215 38.572603, -113.115956 38.572663, -113.11686 38.572612, -113.189536 38.57273199999999, -113.191158 38.57261099999999, -114.05015385888 38.5729744583009, -114.049883347847 38.677365))'
utah_shape = shapely.wkt.loads(utah_wkt)
landsat_qf = LandsatQueryFilters()
# cloud cover less than 30%
landsat_qf.cloud_cover.set_range(end=30)
landsat_qf.aoi.set_geometry(utah_shape.wkb)
# sort by date, with most recent first
landsat_qf.acquired.sort_by(epl_imagery_pb2.DESCENDING)
metadata_servce = MetadataService()
rows = metadata_servce.search_layer_group(data_filters=landsat_qf,
satellite_id=SpacecraftID.LANDSAT_8)
self.assertIsNotNone(rows)
metadata_set = list(rows)
self.assertGreaterEqual(len(metadata_set), 1)
class TestGCPPixelFunctions(unittest.TestCase):
m_metadata = None
base_mount_path = '/imagery'
metadata_service = MetadataService()
iowa_polygon = None
metadata_set = []
r = requests.get("https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json")
taos_geom = r.json()
taos_shape = shapely.geometry.shape(taos_geom['features'][0]['geometry'])
def setUp(self):
metadata_service = MetadataService()
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
# sql_filters = ['scene_id="LC80400312016103LGN00"']
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80400312016103LGN00")
landsat_filters.aoi.set_bounds(*bounding_box)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=1,
data_filters=landsat_filters)
rows = list(rows)
self.m_metadata = rows[0]
wkt_iowa = "POLYGON((-93.76075744628906 42.32707774458643,-93.47854614257812 42.32707774458643," \
"-93.47854614257812 42.12674735753131,-93.76075744628906 42.12674735753131," \
"-93.76075744628906 42.32707774458643))"
self.iowa_polygon = loads(wkt_iowa)
gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', "YES")
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.aoi.set_bounds(*self.taos_shape.bounds)
landsat_filters.acquired.set_range(start=d_start, end=d_end)
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8,
limit=10,
data_filters=landsat_filters)
rows = list(rows)
for row in rows:
self.metadata_set.append(row)
def test_pixel_1(self):
metadata = self.m_metadata
landsat = Landsat(metadata) # , gsurl[2])
code = """import numpy as np
def multiply_rounded(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize,
raster_ysize, buf_radius, gt, **kwargs):
factor = float(kwargs['factor'])
out_ar[:] = np.round_(np.clip(in_ar[0] * factor,0,255))"""
function_arguments = {"factor": "1.5"}
pixel_function_details = FunctionDetails(name="multiply_rounded", band_definitions=[2],
data_type=DataType.FLOAT32, code=code,
arguments=function_arguments)
vrt = landsat.get_vrt([pixel_function_details, 3, 2])
with open('pixel_1.vrt', 'r') as myfile:
data = myfile.read()
expected = etree.XML(data)
actual = etree.XML(vrt)
result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10})
self.assertTrue(result, message)
def test_pixel_ndvi(self):
"""
http://grindgis.com/blog/vegetation-indices-arcgis
NDVI = (NIR - RED) / (NIR + RED)
NDVI = (5 - 4) / (5 + 4)
:return:
"""
landsat = Landsat(self.m_metadata) # , gsurl[2])
code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
with np.errstate(divide = 'ignore', invalid = 'ignore'):
output = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
output[np.isnan(output)] = 0.0
# shift range from -1.0-1.0 to 0.0-2.0
output += 1.0
# scale up from 0.0-2.0 to 0 to 255 by multiplying by 255/2
# https://stackoverflow.com/a/1735122/445372
output *= 65535/2.0
# https://stackoverflow.com/a/10622758/445372
# in place type conversion
out_ar[:] = output.astype(np.int16, copy=False)"""
pixel_function_details = FunctionDetails(name="ndvi_numpy", band_definitions=[4, 5],
data_type=DataType.UINT16, code=code)
vrt = landsat.get_vrt([pixel_function_details, 3, 2])
with open('ndvi_numpy.vrt', 'r') as myfile:
data = myfile.read()
expected = etree.XML(data)
actual = etree.XML(vrt)
result, message = xml_compare(expected, actual, {"GeoTransform": 1e-10})
self.assertTrue(result, message)
gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', "YES")
ds = gdal.Open(vrt)
self.assertIsNotNone(ds)
arr_ndvi = ds.GetRasterBand(1).ReadAsArray()
ds = None
self.assertIsNotNone(arr_ndvi)
scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]]
band_definitions = [pixel_function_details, 3, 2]
nda = landsat.fetch_imagery_array(band_definitions, scale_params)
self.assertIsNotNone(nda)
@staticmethod
def ndvi_numpy(nir, red):
with np.errstate(divide='ignore', invalid='ignore'):
out_ar = np.divide((nir.astype(float) - red.astype(float)), (nir.astype(float) + red.astype(float)))
out_ar[np.isnan(out_ar)] = 0.0
return out_ar
def test_iowa_ndarray(self):
d_start = date(2016, 4, 4)
d_end = date(2016, 8, 7)
bounding_box = self.iowa_polygon.bounds
# sql_filters = ["cloud_cover<=15"]
landsat_filters = LandsatQueryFilters()
landsat_filters.cloud_cover.set_range(end=15, end_inclusive=15)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
metadata = rows[0]
landsat = Landsat(metadata)
code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
with np.errstate(divide = 'ignore', invalid = 'ignore'):
out_ar[:] = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
out_ar[np.isnan(out_ar)] = 0.0
out_ar """
pixel_function_details = FunctionDetails(name="ndvi_numpy", band_definitions=[4, 5], code=code, data_type=DataType.FLOAT32)
# pixel_function_details = {
# "band_numbers": [4, 5],
# "function_code": code,
# "function_type": "ndvi_numpy",
# "data_type": DataType.FLOAT32,
# }
band_definitions = [pixel_function_details, 4, 5]
vrt = landsat.get_vrt(band_definitions)
ds = gdal.Open(vrt)
self.assertIsNotNone(ds)
arr_4 = ds.GetRasterBand(2).ReadAsArray()
arr_5 = ds.GetRasterBand(3).ReadAsArray()
arr_ndvi = ds.GetRasterBand(1).ReadAsArray()
del ds
del landsat
print(np.ndarray.max(arr_ndvi))
print(np.ndarray.min(arr_ndvi))
self.assertFalse(np.any(np.isinf(arr_ndvi)))
self.assertIsNotNone(arr_ndvi)
local_ndvi = self.ndvi_numpy(arr_5, arr_4)
del arr_4
del arr_5
self.assertFalse(np.any(np.isinf(local_ndvi)))
np.testing.assert_almost_equal(arr_ndvi, local_ndvi)
def test_iowa_scaled(self):
d_start = date(2016, 4, 4)
d_end = date(2016, 8, 7)
bounding_box = self.iowa_polygon.bounds
# sql_filters = ["cloud_cover<=15"]
landsat_filters = LandsatQueryFilters()
landsat_filters.cloud_cover.set_range(end=15, end_inclusive=True)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
metadata = rows[0]
landsat = Landsat(metadata)
code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
with np.errstate(divide = 'ignore', invalid = 'ignore'):
factor = float(kwargs['factor'])
out_ar[:] = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
out_ar[np.isnan(out_ar)] = 0.0
# shift range from -1.0-1.0 to 0.0-2.0
out_ar += 1.0
# scale up from 0.0-2.0 to 0 to 255 by multiplying by 255/2
out_ar *= factor/2.0"""
# pixel_function_details = {
# "function_arguments": {"factor": 255},
# "band_numbers": [4, Band.NIR],
# "function_code": code,
# "function_type": "ndvi_numpy",
# "data_type": DataType.FLOAT32,
# }
pixel_function_details = FunctionDetails(name="ndvi_numpy",
band_definitions=[4, Band.NIR],
code=code, arguments={"factor": 255},
data_type=DataType.FLOAT32)
band_definitions = [pixel_function_details, Band.RED, 5]
vrt = landsat.get_vrt(band_definitions)
ds = gdal.Open(vrt)
self.assertIsNotNone(ds)
arr_4 = ds.GetRasterBand(2).ReadAsArray()
arr_5 = ds.GetRasterBand(3).ReadAsArray()
arr_ndvi = ds.GetRasterBand(1).ReadAsArray()
del ds
del landsat
print(np.ndarray.max(arr_ndvi))
# print(np.ndarray.min(arr_ndvi))
# self.assertFalse(np.any(np.isinf(arr_ndvi)))
self.assertIsNotNone(arr_ndvi)
local_ndvi = self.ndvi_numpy(arr_5, arr_4)
del arr_4
del arr_5
local_ndvi += 1.0
local_ndvi *= float(pixel_function_details.arguments['factor']) / 2.0
self.assertFalse(np.any(np.isinf(local_ndvi)))
np.floor(arr_ndvi, out=arr_ndvi)
np.floor(local_ndvi, out=local_ndvi)
np.testing.assert_almost_equal(arr_ndvi, local_ndvi, decimal=0)
def test_malformed_funciton(self):
code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
with np.errstate(divide = 'ignore', invalid = 'ignore'):
factor = float(kwargs['factor'])
out_ar[:] = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
out_ar[np.isnan(out_ar)] = 0.0
# shift range from -1.0-1.0 to 0.0-2.0
out_ar += 1.0
# scale up from 0.0-2.0 to 0 to 255 by multiplying by 255/2
out_ar *= factor/2.0 """
# pixel_function_details = {
# "function_arguments": {"factor": 255},
# "band_numbers": [4, 5],
# "function_code": code,
# "function_type": "ndvi_numpy",
# "data_type": DataType.FLOAT32,
# }
self.assertRaises(py_compile.PyCompileError, lambda: FunctionDetails(name="ndvi_numpy",
code=code,
band_definitions=[4, 5],
data_type=DataType.FLOAT32,
arguments={"factor": 255}))
# def test_translate_vrt(self):
# # LC80390332016208LGN00
"""
gdalbuildvrt -vrtnodata 0 0 0 -separate rgb_35.vrt /imagery/gcp-public-data-landsat/LC08/PRE/033/035/LC80330352017072LGN00/LC80330352017072LGN00_B4.TIF /imagery/gcp-public-data-landsat/LC08/PRE/033/035/LC80330352017072LGN00/LC80330352017072LGN00_B3.TIF /imagery/gcp-public-data-landsat/LC08/PRE/033/035/LC80330352017072LGN00/LC80330352017072LGN00_B2.TIF
gdalbuildvrt -separate rgb_34.vrt /imagery/gcp-public-data-landsat/LC08/PRE/033/034/LC80330342017072LGN00/LC80330342017072LGN00_B4.TIF /imagery/gcp-public-data-landsat/LC08/PRE/033/034/LC80330342017072LGN00/LC80330342017072LGN00_B3.TIF /imagery/gcp-public-data-landsat/LC08/PRE/033/034/LC80330342017072LGN00/LC80330342017072LGN00_B2.TIF
"""
# # gdal_translate -of VRT -ot Byte -scale -tr 60 60 rgb.vrt rgb_byte_scaled.vrt
#
# self.assertTrue(True)
# sql_filters = ['scene_id="LC80330342017072LGN00"']
# metadata_service = MetadataService()
# rows = metadata_service.search(SpacecraftID.LANDSAT_8, data_filters=landsat_filters)
#rows = list(rows)
#
#
# metadata = rows[0]
# gsurl = urlparse(metadata.base_url)
# storage = Storage(gsurl[1])
#
# b_mounted = storage.mount_sub_folder(gsurl[2], self.base_mount_path)
# landsat = Landsat(base_mount_path, gsurl[2])
# vrt = landsat.get_vrt(metadata, [5, 4, 3])
#
# with open('gdalbuildvrt_LC80390332016208LGN00.vrt', 'r') as myfile:
# data = myfile.read()
# expected = etree.XML(data)
# actual = etree.XML(vrt)
# result, message = xml_compare(expected, actual)
# self.assertTrue(result, message)
def test_ndvi_taos(self):
code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
with np.errstate(divide = 'ignore', invalid = 'ignore'):
factor = float(kwargs['factor'])
output = np.divide((in_ar[1] - in_ar[0]), (in_ar[1] + in_ar[0]))
output[np.isnan(output)] = 0.0
# shift range from -1.0-1.0 to 0.0-2.0
output += 1.0
# scale up from 0.0-2.0 to 0 to 255 by multiplying by 255/2
# https://stackoverflow.com/a/1735122/445372
output *= factor/2.0
# https://stackoverflow.com/a/10622758/445372
# in place type conversion
out_ar[:] = output.astype(np.int16, copy=False)"""
code2 = """import numpy as np
def ndvi_numpy2(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
with np.errstate(divide = 'ignore', invalid = 'ignore'):
output = (in_ar[1] - in_ar[0]) / (in_ar[1] + in_ar[0])
output[np.isnan(output)] = 0.0
out_ar[:] = output"""
landsat = Landsat(self.metadata_set)
scale_params = [[0, DataType.UINT16.range_max, -1.0, 1.0]]
pixel_function_details = FunctionDetails(name="ndvi_numpy",
band_definitions=[Band.RED, Band.NIR],
code=code,
arguments={"factor": DataType.UINT16.range_max},
data_type=DataType.UINT16)
gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', "YES")
nda = landsat.fetch_imagery_array([pixel_function_details],
scale_params=scale_params,
polygon_boundary_wkb=self.taos_shape.wkb,
output_type=DataType.FLOAT32)
self.assertIsNotNone(nda)
self.assertGreaterEqual(1.0, nda.max())
self.assertLessEqual(-1.0, nda.min())
pixel_function_details = FunctionDetails(name="ndvi_numpy2",
band_definitions=[Band.RED, Band.NIR],
code=code2,
data_type=DataType.FLOAT32)
nda2 = landsat.fetch_imagery_array([pixel_function_details],
polygon_boundary_wkb=self.taos_shape.wkb,
output_type=DataType.FLOAT32)
self.assertIsNotNone(nda2)
self.assertGreaterEqual(1.0, nda2.max())
self.assertLessEqual(-1.0, nda2.min())
def test_fail_1_to_1(self):
code = """import numpy as np
def ndvi_numpy(in_ar, out_ar, xoff, yoff, xsize, ysize, raster_xsize, raster_ysize, buf_radius, gt, **kwargs):
out_ar[:] = in_ar[0]"""
landsat = Landsat(self.metadata_set)
scale_params = [[0, 40000], [0, 40000], [0, 40000]]
pixel_function_details = FunctionDetails(name="ndvi_numpy",
band_definitions=[Band.RED],
code=code,
arguments={"factor": DataType.UINT16.range_max},
data_type=DataType.UINT16)
gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', "YES")
nda = landsat.fetch_imagery_array([pixel_function_details, Band.GREEN, Band.BLUE],
scale_params=scale_params,
polygon_boundary_wkb=self.taos_shape.wkb,
output_type=DataType.BYTE)
nda2 = landsat.fetch_imagery_array([Band.RED, Band.GREEN, Band.BLUE],
scale_params=scale_params,
polygon_boundary_wkb=self.taos_shape.wkb,
output_type=DataType.BYTE)
self.assertIsNotNone(nda)
np.testing.assert_almost_equal(nda, nda2)
np.testing.assert_equal(nda, nda2)
@unittest.skip("failing for some reason. unknown.")
def test_native_vs_custom(self):
landsat = Landsat(self.metadata_set)
gdal.SetConfigOption('GDAL_VRT_ENABLE_PYTHON', "YES")
pixel_native = FunctionDetails(name="sqrt",
band_definitions=[Band.RED],
data_type=DataType.UINT16,
transfer_type=DataType.FLOAT32)
nda = landsat.fetch_imagery_array([pixel_native],
polygon_boundary_wkb=self.taos_shape.wkb,
output_type=DataType.FLOAT32)
self.assertIsNotNone(nda)