def test_aws_file_path(self):
# PRE s3://landsat-pds/L8/139/045/LC81390452014295LGN00/
# non-PRE s3://landsat-pds/c1/L8/139/045/LC08_L1TP_139045_20170304_20170316_01_T1/
metadataservice = MetadataService()
start_date = datetime.datetime.strptime('14295', '%y%j').date()
landsat_filters = LandsatQueryFilters()
landsat_filters.wrs_path.set_value(139)
landsat_filters.wrs_row.set_value(45)
landsat_filters.acquired.set_range(start_date, True, start_date, True)
landsat_filters.collection_number.set_value("PRE")
rows = metadataservice.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
# turn gernator into list
metadata_set = list(rows)
self.assertEqual(len(metadata_set), 1)
metadata = metadata_set[0]
self.assertEqual(metadata.get_aws_file_path(), "/imagery/L8/139/045/LC81390452014295LGN00")
landsat_filters = LandsatQueryFilters()
landsat_filters.wrs_path.set_value(139)
landsat_filters.wrs_row.set_value(45)
# landsat_filters.collection_number.set_exclude_value("PRE")
landsat_filters.acquired.set_range(start=date(2017, 3, 4), end=date(2017, 3, 4))
rows = metadataservice.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
metadata_set = list(rows)
self.assertEqual(len(metadata_set), 2)
metadata = metadata_set[0]
self.assertEqual(metadata.get_aws_file_path(),
"/imagery/c1/L8/139/045/LC08_L1TP_139045_20170304_20170316_01_T1")
metadata = metadata_set[1]
self.assertEqual(metadata.get_aws_file_path(),
"/imagery/L8/139/045/LC81390452017063LGN00")
def setUp(self):
metadata_service = MetadataService()
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999,
44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80400312016103LGN00")
landsat_filters.acquired.set_range(start=d_start, end=d_end)
landsat_filters.aoi.set_bounds(*bounding_box)
# sql_filters = ['scene_id="LC80400312016103LGN00"']
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=1,
data_filters=landsat_filters)
rows = list(rows)
self.m_row_data = rows[0]
wkt_iowa = "POLYGON((-93.76075744628906 42.32707774458643,-93.47854614257812 42.32707774458643," \
"-93.47854614257812 42.12674735753131,-93.76075744628906 42.12674735753131," \
"-93.76075744628906 42.32707774458643))"
self.iowa_polygon = loads(wkt_iowa)
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.acquired.set_range(start=d_start, end=d_end)
landsat_filters.aoi.set_bounds(*self.taos_shape.bounds)
# sql_filters = ['collection_number="PRE"']
rows = self.metadata_service.search(SpacecraftID.LANDSAT_8,
limit=10,
data_filters=landsat_filters)
for row in rows:
self.metadata_set.append(row)
def test_dates(self):
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.acquired.set_range(d_start, True, d_end, True)
result = landsat_filters.get_sql()
expected = "{}{}".format(expected_prefix, ' WHERE (((t1.sensing_time >= "2017-03-12T00:00:00") AND (t1.sensing_time <= "2017-03-19T23:59:59.999999")) AND (t1.collection_number IN ("PRE"))) LIMIT 10')
self.maxDiff = None
self.assertMultiLineEqual(expected, result)
d_start = date(2017, 3, 24) # 2017-03-12
landsat_filters.acquired.set_range(start=d_start, start_inclusive=True)
result = landsat_filters.get_sql()
expected = "{}{}".format(expected_prefix, ' WHERE ((((t1.sensing_time >= "2017-03-12T00:00:00") AND (t1.sensing_time <= "2017-03-19T23:59:59.999999")) OR (t1.sensing_time >= "2017-03-24T00:00:00")) AND (t1.collection_number IN ("PRE"))) LIMIT 10')
self.maxDiff = None
self.assertMultiLineEqual(expected, result)
d_end = date(2017, 3, 9)
landsat_filters.acquired.set_range(end=d_end, end_inclusive=True)
result = landsat_filters.get_sql()
expected = "{}{}".format(expected_prefix, ' WHERE (((((t1.sensing_time >= "2017-03-12T00:00:00") AND (t1.sensing_time <= "2017-03-19T23:59:59.999999")) OR (t1.sensing_time >= "2017-03-24T00:00:00")) OR (t1.sensing_time <= "2017-03-09T23:59:59.999999")) AND (t1.collection_number IN ("PRE"))) LIMIT 10')
self.maxDiff = None
self.assertMultiLineEqual(expected, result)
def test_peewee_1(self):
a = LandsatQueryFilters()
a.scene_id.set_value("LC80270312016188LGN00")
# sql_message = a.scene_id.get()
sql_stuff = a.get_sql()
self.assertIsNotNone(sql_stuff)
self.maxDiff = None
expected = "{}{}".format(expected_prefix, ' WHERE (t1.scene_id IN ("LC80270312016188LGN00")) LIMIT 10')
self.assertMultiLineEqual(expected, sql_stuff)
def test_dates_2(self):
d_start = date(2017, 3, 12) # 2017-03-12
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.acquired.set_value(d_start)
result = landsat_filters.get_sql()
expected = "{}{}".format(expected_prefix, ' WHERE (((t1.sensing_time >= "2017-03-12T00:00:00") AND (t1.sensing_time <= "2017-03-12T23:59:59.999999")) AND (t1.collection_number IN ("PRE"))) LIMIT 10')
self.maxDiff = None
self.assertMultiLineEqual(expected, result)
def test_or_and(self):
lqf = LandsatQueryFilters()
lqf.cloud_cover.set_exclude_value(44)
lqf.cloud_cover.set_range(end=42, end_inclusive=True)
lqf.cloud_cover.set_range(start=46, start_inclusive=True)
sql = lqf.get_sql()
expected = "{}{}".format(expected_prefix, " WHERE (NOT (t1.cloud_cover IN (44.0)) AND ((t1.cloud_cover <= 42.0) OR (t1.cloud_cover >= 46.0))) LIMIT 10")
self.maxDiff = None
self.assertMultiLineEqual(expected, sql)
def test_dates_4(self):
# TODO include in metadata tests and check dates (should only be the 10th and the 17th
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.acquired.set_range(start=date(2017, 3, 10), end=date(2017, 3, 17))
landsat_filters.acquired.set_exclude_range(start=date(2017, 3, 11), end=date(2017, 3, 16))
result = landsat_filters.get_sql()
expected = "{}{}".format(expected_prefix, ' WHERE ((((t1.sensing_time >= "2017-03-10T00:00:00") AND (t1.sensing_time <= "2017-03-17T23:59:59.999999")) AND NOT ((t1.sensing_time >= "2017-03-11T00:00:00") AND (t1.sensing_time <= "2017-03-16T23:59:59.999999"))) AND (t1.collection_number IN ("PRE"))) LIMIT 10')
self.maxDiff = None
self.assertMultiLineEqual(expected, result)
def test_select_all(self):
expected_prefix = 'SELECT * FROM [bigquery-public-data:cloud_storage_geo_index.landsat_index] AS t1 '
a = LandsatQueryFilters()
# sql_message = a.scene_id.get()
sql_stuff = a.get_sql()
self.assertIsNotNone(sql_stuff)
self.maxDiff = None
expected = "{}{}".format(expected_prefix, 'LIMIT 10')
self.assertMultiLineEqual(expected, sql_stuff)
def test_satellite_id(self):
landsat_filter = LandsatQueryFilters()
landsat_filter.spacecraft_id.set_value(SpacecraftID.LANDSAT_4.name)
sql_stuff = landsat_filter.get_sql()
expected = "{}{}".format(expected_prefix, ' WHERE (t1.spacecraft_id IN ("LANDSAT_4")) LIMIT 10')
self.assertMultiLineEqual(expected, sql_stuff)
# don't allow repeat adds
landsat_filter.spacecraft_id.set_value(SpacecraftID.LANDSAT_4.name)
sql_stuff = landsat_filter.get_sql()
self.assertMultiLineEqual(expected, sql_stuff)
def test_peewee_1(self):
a = LandsatQueryFilters()
a.scene_id.set_value("LC80270312016188LGN00")
sql_stuff = a.get_sql()
client = bigquery.Client()
query = client.run_sync_query(sql_stuff)
query.timeout_ms = 1000
query.run()
for row in query.rows:
metadata = Metadata(row)
self.assertEqual(metadata.scene_id, "LC80270312016188LGN00")
def test_bounding_box(self):
d_start = date(2017, 6, 24)
d_end = date(2017, 9, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
# sql_filters = ['collection_number!="PRE"']
landsat_filter = LandsatQueryFilters()
landsat_filter.collection_number.set_exclude_value("PRE")
landsat_filter.acquired.set_range(d_start, True, d_end, True)
landsat_filter.aoi.set_bounds(*bounding_box)
landsat_filter.acquired.sort_by(epl_imagery_pb2.DESCENDING)
result = landsat_filter.get_sql()
expected = "{}{}".format(expected_prefix, ' WHERE ((((t1.sensing_time >= "2017-06-24T00:00:00") AND (t1.sensing_time <= "2017-09-24T23:59:59.999999")) AND ((((t1.west_lon >= -115.927734375) AND (t1.west_lon <= -78.31054687499999)) OR ((t1.west_lon <= -115.927734375) AND (t1.east_lon >= -115.927734375))) AND (((t1.south_lat <= 34.52466147177172) AND (t1.north_lat >= 34.52466147177172)) OR ((t1.south_lat > 34.52466147177172) AND (t1.south_lat <= 44.84029065139799))))) AND NOT (t1.collection_number IN ("PRE"))) ORDER BY t1.sensing_time DESC LIMIT 10')
self.maxDiff = None
self.assertMultiLineEqual(expected, result)
def test_pixel_function_vrt_1(self):
utah_box = (-112.66342163085938, 37.738141282210385,
-111.79824829101562, 38.44821130413263)
d_start = date(2016, 7, 20)
d_end = date(2016, 7, 28)
landsat_filter = LandsatQueryFilters()
landsat_filter.collection_number.set_value("PRE")
landsat_filter.cloud_cover.set_range(
end=5, end_inclusive=True
) #landsat_filter.cloud_cover.set_range_end(5, True)
landsat_filter.acquired.set_range(d_start, True, d_end, True)
landsat_filter.aoi.set_bounds(*utah_box)
rows = self.metadata_service.search(SpacecraftID.LANDSAT_8,
limit=10,
data_filters=landsat_filter)
rows = list(rows)
self.assertEqual(0, len(rows))
d_end = date(2016, 8, 28)
landsat_filter = LandsatQueryFilters()
landsat_filter.collection_number.set_value("PRE")
landsat_filter.cloud_cover.set_range(
end=5, end_inclusive=False
) #landsat_filter.cloud_cover.set_range_end(5, False)
landsat_filter.acquired.set_range(
end=d_end, end_inclusive=True
) #landsat_filter.acquired.set_range_end(d_end, True)
landsat_filter.acquired.sort_by(epl_imagery_pb2.DESCENDING)
landsat_filter.aoi.set_bounds(*utah_box)
rows = self.metadata_service.search(SpacecraftID.LANDSAT_8,
limit=10,
data_filters=landsat_filter)
rows = list(rows)
self.assertEqual(len(rows), 10)
# metadata_row = ['LC80390332016208LGN00', '', 'LANDSAT_8', 'OLI_TIRS', '2016-07-26',
# '2016-07-26T18:14:46.9465460Z', 'PRE', 'N/A', 'L1T', 39, 33, 1.69,
# 39.96962, 37.81744, -115.27267, -112.56732, 1070517542,
# 'gs://gcp-public-data-landsat/LC08/PRE/039/033/LC80390332016208LGN00']
metadata = rows[0]
# GDAL helper functions for generating VRT
landsat = Landsat([metadata])
# get a numpy.ndarray from bands for specified imagery
band_numbers = [4, 3, 2]
scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]]
nda = landsat.fetch_imagery_array(band_numbers, scale_params)
self.assertEqual(nda.shape, (3876, 3806, 3))
def test_not_in_in(self):
lqf = LandsatQueryFilters()
lqf.cloud_cover.set_exclude_value(44)
lqf.cloud_cover.set_exclude_value(48)
lqf.cloud_cover.set_exclude_value(50)
lqf.cloud_cover.set_value(64)
lqf.cloud_cover.set_value(68)
lqf.cloud_cover.set_value(70)
lqf.east_lon.sort_by(epl_imagery_pb2.DESCENDING)
lqf.cloud_cover.set_range(end=99, end_inclusive=False, start=42)
sql = lqf.get_sql()
expected = "{}{}".format(expected_prefix, " WHERE (((t1.cloud_cover IN (64.0, 68.0, 70.0)) AND NOT (t1.cloud_cover IN (44.0, 48.0, 50.0))) AND ((t1.cloud_cover >= 42.0) AND (t1.cloud_cover < 99.0))) ORDER BY t1.east_lon DESC LIMIT 10")
self.maxDiff = None
self.assertMultiLineEqual(expected, sql)
def test_clear_bounds(self):
d_start = date(2017, 6, 24)
d_end = date(2017, 9, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
# sql_filters = ['collection_number!="PRE"']
landsat_filter = LandsatQueryFilters()
landsat_filter.collection_number.set_exclude_value("PRE")
landsat_filter.acquired.set_range(d_start, True, d_end, True)
expected = landsat_filter.get_sql()
landsat_filter.aoi.set_bounds(*bounding_box)
landsat_filter.aoi.query_params.ClearField("bounds")
actual = landsat_filter.get_sql()
self.maxDiff = None
self.assertMultiLineEqual(expected, actual)
def test_bounds_repeat_bug(self):
# grab the Taos, NM county outline from a geojson hosted on github
r = requests.get(
"https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json"
)
taos_geom = r.json()
taos_shape = shapely.geometry.shape(
taos_geom['features'][0]['geometry'])
metadata_service = MetadataService()
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # epl api is inclusive
# PRE is a collection type that specifies certain QA standards
# sql_filters = ['collection_number="PRE"']
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.acquired.set_range(start=d_start, end=d_end)
landsat_filters.aoi.set_bounds(*taos_shape.bounds)
# search the satellite metadata for images of Taos withing the given date range
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=10,
data_filters=landsat_filters)
# group the scenes together in a list
for row in rows:
self.assertEqual(4, len(row.bounds))
def test_bounding_box_1(self):
metadata_service = MetadataService()
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999,
44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.aoi.set_bounds(*bounding_box)
landsat_filters.acquired.set_range(start=d_start, end=d_end)
metadata_rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
metadata_rows = list(metadata_rows)
self.assertEqual(len(metadata_rows), 10)
for row in metadata_rows:
self.assertEqual(row.spacecraft_id, SpacecraftID.LANDSAT_8)
d_actual = datetime.strptime(row.date_acquired, '%Y-%m-%d').date()
self.assertLessEqual(d_actual, d_end)
self.assertGreaterEqual(d_actual, d_start)
test_box = row.bounds
self.assertTrue(
(bounding_box[0] < test_box[2] < bounding_box[2])
or (bounding_box[0] < test_box[0] < bounding_box[2]))
self.assertTrue(
(bounding_box[1] < test_box[3] < bounding_box[3])
or (bounding_box[1] < test_box[1] < bounding_box[3]))
def test_cloud_sort(self):
# gs://gcp-public-data-landsat/LC08/PRE/044/034/LC80440342016259LGN00/
metadata_service = MetadataService()
d = date(2016, 6, 24)
landsat_filters = LandsatQueryFilters()
landsat_filters.acquired.set_range(d, True)
landsat_filters.wrs_path.set_value(125)
landsat_filters.wrs_row.set_value(49)
landsat_filters.acquired.set_range(end=d, end_inclusive=True)
landsat_filters.cloud_cover.sort_by(epl_imagery_pb2.ASCENDING)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 10)
last_cloud = 0
for row in rows:
# test Order by
self.assertLessEqual(last_cloud, row.cloud_cover)
last_cloud = row.cloud_cover
landsat_filters.cloud_cover.sort_by(epl_imagery_pb2.DESCENDING)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 10)
last_cloud = 100
for row in rows:
# test Order by
self.assertGreaterEqual(last_cloud, row.cloud_cover)
last_cloud = row.cloud_cover
def test_end_date(self):
r = requests.get(
"https://raw.githubusercontent.com/johan/world.geo.json/master/countries/BEL.geo.json"
)
area_geom = r.json()
area_shape = shapely.geometry.shape(
area_geom['features'][0]['geometry'])
# gs://gcp-public-data-landsat/LC08/PRE/044/034/LC80440342016259LGN00/
metadata_service = MetadataService()
d = date(2016, 6, 24)
landsat_filter = LandsatQueryFilters()
landsat_filter.acquired.set_range(end=d, end_inclusive=True)
landsat_filter.acquired.sort_by(epl_imagery_pb2.DESCENDING)
landsat_filter.aoi.set_bounds(*area_shape.bounds)
rows = metadata_service.search(SpacecraftID.LANDSAT_7,
data_filters=landsat_filter)
rows = list(rows)
self.assertEqual(len(rows), 10)
d_previous = d
for row in rows:
self.assertEqual(row.spacecraft_id, SpacecraftID.LANDSAT_7)
d_actual = datetime.strptime(row.date_acquired, '%Y-%m-%d').date()
self.assertLessEqual(d_actual, d_previous)
d_previous = d_actual
def test_add_metadata_error(self):
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.data_type.set_value("L1T")
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(SpacecraftID.LANDSAT_8,
limit=2,
data_filters=landsat_filters)
rows = list(rows)
metadata_1 = rows[0]
metadata_2 = rows[1]
bands = [Band.RED, Band.BLUE, Band.GREEN]
band_map = BandMap(SpacecraftID.LANDSAT_8)
raster_metadata = RasterMetadata()
storage = Storage()
storage.mount_sub_folder(metadata_1)
storage.mount_sub_folder(metadata_2)
second = False
for band in bands:
band_number = band_map.get_number(band)
if second:
self.assertRaises(Exception, lambda: raster_metadata.add_metadata(band_number, metadata_2))
raster_metadata.add_metadata(band_number, metadata_1)
second = True
def test_metatdata_file_list(self):
wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \
"138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \
"2469482421875 -27.57843813308233))"
polygon = loads(wkt)
metadata_service = MetadataService()
# sql_filters = ['cloud_cover=0']
d_start = date(2006, 8, 4)
d_end = date(2006, 8, 5)
bounding_box = polygon.bounds
# sql_filters = ['wrs_row=79']
landsat_filters = LandsatQueryFilters()
# landsat_filters.wrs_path.set_value(125)
landsat_filters.wrs_row.set_value(79)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = metadata_service.search(
SpacecraftID.LANDSAT_5,
data_filters=landsat_filters)
rows = list(rows)
metadata = rows[0]
self.assertEqual(len(metadata.get_file_list()), 0)
def test_unmount_destructor_conflict(self):
wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \
"138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \
"2469482421875 -27.57843813308233))"
polygon = loads(wkt)
# sql_filters = ['cloud_cover=0']
d_start = date(2006, 8, 4)
d_end = date(2006, 8, 7)
bounding_box = polygon.bounds
# sql_filters = ['wrs_row=79']
landsat_filters = LandsatQueryFilters()
landsat_filters.wrs_row.set_value(79)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_5,
data_filters=landsat_filters)
rows = list(rows)
metadata = rows[0]
landsat = Landsat(metadata)
vrt = landsat.get_vrt([4])
storage = Storage("gcp-public-data-landsat")
landsat_2 = Landsat(metadata)
vrt = landsat_2.get_vrt([4])
del landsat
self.assertTrue(storage.is_mounted(metadata))
def test_australia(self):
# 5th Place: Lake Eyre Landsat 5 Acquired August 5, 2006
wkt = "POLYGON((136.2469482421875 -27.57843813308233,138.6639404296875 -27.57843813308233," \
"138.6639404296875 -29.82351878748485,136.2469482421875 -29.82351878748485,136." \
"2469482421875 -27.57843813308233))"
polygon = loads(wkt)
# sql_filters = ['cloud_cover=0']
d_start = date(2006, 8, 4)
d_end = date(2006, 8, 7)
bounding_box = polygon.bounds
landsat_filters = LandsatQueryFilters()
landsat_filters.wrs_row.set_value(79)
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = self.metadata_service.search(
SpacecraftID.LANDSAT_5,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 1)
metadata = rows[0]
landsat = Landsat(metadata)
# get a numpy.ndarray from bands for specified imagery
band_numbers = [3, 2, 1]
scale_params = [[0.0, 65535], [0.0, 65535], [0.0, 65535]]
# nda = landsat.__get_ndarray(band_numbers, metadata, scale_params)
nda = landsat.fetch_imagery_array(band_numbers, scale_params)
self.assertEqual((3581, 4046, 3), nda.shape)
def test_cloud_cover(self):
metadata_service = MetadataService()
# TODO figure out what's wrong here for AWS
self.assertTrue(True)
# sql_filters = ['cloud_cover=0']
d_start = date(2015, 6, 24)
d_end = date(2016, 6, 24)
bounding_box = (-115.927734375, 34.52466147177172, -78.31054687499999, 44.84029065139799)
landsat_filters = LandsatQueryFilters()
landsat_filters.cloud_cover.set_value(0)
landsat_filters.acquired.set_range(start=d_start, end=d_end)
landsat_filters.aoi.set_bounds(*bounding_box)
rows = metadata_service.search(
SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 10)
for row in rows:
self.assertEqual(row.spacecraft_id, SpacecraftID.LANDSAT_8)
d_actual = datetime.strptime(row.date_acquired, '%Y-%m-%d').date()
self.assertLessEqual(d_actual, d_end)
self.assertGreaterEqual(d_actual, d_start)
test_box = row.bounds
self.assertTrue(
(bounding_box[0] < test_box[2] < bounding_box[2]) or
(bounding_box[0] < test_box[0] < bounding_box[2]))
self.assertTrue(
(bounding_box[1] < test_box[3] < bounding_box[3]) or
(bounding_box[1] < test_box[1] < bounding_box[3]))
def search(
self,
satellite_id: SpacecraftID=None,
limit=10,
data_filters: LandsatQueryFilters=None):
if GRPC_SERVICE_HOST == "localhost" or ip_reg.match(GRPC_SERVICE_HOST):
channel = grpc.insecure_channel(IMAGERY_SERVICE, options=GRPC_CHANNEL_OPTIONS)
else:
channel_credentials = grpc.ssl_channel_credentials()
channel = grpc.secure_channel(IMAGERY_SERVICE, credentials=channel_credentials, options=GRPC_CHANNEL_OPTIONS)
#
stub = epl_imagery_pb2_grpc.ImageryOperatorsStub(channel)
data_filters_pb = None
if data_filters:
data_filters_pb = data_filters.get_query_filter()
request = epl_imagery_pb2.MetadataRequest(satellite_id=satellite_id,
limit=limit,
data_filters=data_filters_pb)
results_generator = stub.MetadataSearch(request)
return results_generator
def test_sort_by_one_field(self):
expected_prefix = 'SELECT * FROM [bigquery-public-data:cloud_storage_geo_index.landsat_index] AS t1 '
a = LandsatQueryFilters()
a.cloud_cover.sort_by(epl_imagery_pb2.ASCENDING)
sql_stuff = a.get_sql()
self.assertIsNotNone(sql_stuff)
self.maxDiff = None
expected = "{}{}".format(expected_prefix, 'ORDER BY t1.cloud_cover LIMIT 10')
self.assertMultiLineEqual(expected, sql_stuff)
a.east_lon.sort_by(epl_imagery_pb2.ASCENDING)
sql_stuff = a.get_sql()
self.assertIsNotNone(sql_stuff)
self.maxDiff = None
expected = "{}{}".format(expected_prefix, 'ORDER BY t1.east_lon LIMIT 10')
self.assertMultiLineEqual(expected, sql_stuff)
def test_polygon_wkb_metadata(self):
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive
self.metadata_service = MetadataService()
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_geometry(self.taos_shape.wkb)
# landsat_filters.geometry_bag.geometry_binaries.append(self.taos_shape.wkb)
metadata_rows = self.metadata_service.search(
SpacecraftID.LANDSAT_8, limit=10, data_filters=landsat_filters)
metadata_set = []
for row in metadata_rows:
metadata_set.append(row)
landsat = Landsat(metadata_set)
band_numbers = [Band.NIR, Band.SWIR1, Band.SWIR2]
scaleParams = [[0.0, 40000.0], [0.0, 40000.0], [0.0, 40000.0]]
nda = landsat.fetch_imagery_array(
band_numbers,
scaleParams,
polygon_boundary_wkb=self.taos_shape.wkb)
self.assertIsNotNone(nda)
self.assertEqual((1804, 1295, 3), nda.shape)
def test_polygon_boundary(self):
d_start = date(2017, 3, 12) # 2017-03-12
d_end = date(2017, 3, 19) # 2017-03-20, epl api is inclusive
r = requests.get(
"https://raw.githubusercontent.com/johan/world.geo.json/master/countries/USA/NM/Taos.geo.json"
)
taos_geom = r.json()
taos_shape = shapely.geometry.shape(
taos_geom['features'][0]['geometry'])
metadata_service = MetadataService()
landsat_filters = LandsatQueryFilters()
landsat_filters.collection_number.set_value("PRE")
landsat_filters.acquired.set_range(d_start, True, d_end, True)
landsat_filters.aoi.set_geometry(taos_shape.wkb)
# landsat_filters.geometry_bag.geometry_binaries.append(taos_shape.wkb)
metadata_rows = metadata_service.search(SpacecraftID.LANDSAT_8,
limit=10,
data_filters=landsat_filters)
# mounted directory in docker container
metadata_set = []
for row in metadata_rows:
metadata_set.append(row)
self.assertEqual(len(metadata_set), 2)
def test_query_param_landsat(self):
a = LandsatQueryFilters()
a.scene_id.set_value("LC80330352017072LGN00")
a.cloud_cover.set_range(end=2, end_inclusive=False)
a.collection_number.set_value("PRE")
a.total_size.sort_by(epl_imagery_pb2.ASCENDING)
a.wrs_path.set_value(33)
result = a.get_sql()
expected = "{}{}".format(expected_prefix, ' WHERE ((((t1.cloud_cover < 2.0) AND (t1.collection_number IN ("PRE"))) AND (t1.scene_id IN ("LC80330352017072LGN00"))) AND (t1.wrs_path IN (33))) ORDER BY t1.total_size LIMIT 10')
self.maxDiff = None
self.assertMultiLineEqual(expected, result)
result = a.get_sql()
expected = "{}{}".format(expected_prefix, ' WHERE ((((t1.cloud_cover < 2.0) AND (t1.collection_number IN ("PRE"))) AND (t1.scene_id IN ("LC80330352017072LGN00"))) AND (t1.wrs_path IN (33))) ORDER BY t1.total_size LIMIT 10')
self.assertMultiLineEqual(expected, result)
def test_start_date(self):
# gs://gcp-public-data-landsat/LC08/PRE/044/034/LC80440342016259LGN00/
metadata_service = MetadataService()
d = date(2016, 6, 24)
landsat_filters = LandsatQueryFilters()
landsat_filters.acquired.set_range(d, True)
landsat_filters.wrs_path.set_value(125)
landsat_filters.wrs_row.set_value(49)
landsat_filters.acquired.sort_by(epl_imagery_pb2.ASCENDING)
landsat_filters.acquired.set_range(end=d, end_inclusive=True)
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 10)
d_previous = datetime.datetime.strptime("1945-01-01",
'%Y-%m-%d').date()
for row in rows:
self.assertEqual(row.spacecraft_id.name,
SpacecraftID.LANDSAT_8.name)
d_actual = datetime.datetime.strptime(row.date_acquired,
'%Y-%m-%d').date()
# test Order by
self.assertGreaterEqual(d_actual, d_previous)
d_previous = d_actual
def test_scene_id(self):
landsat_filters = LandsatQueryFilters()
landsat_filters.scene_id.set_value("LC80270312016188LGN00")
metadata_service = MetadataService()
rows = metadata_service.search(SpacecraftID.LANDSAT_8,
data_filters=landsat_filters)
rows = list(rows)
self.assertEqual(len(rows), 1)