def test_api_query_format():
wkt = 'POLYGON((0 0,1 1,0 1,0 0))'
now = datetime.now()
last_24h = format_query_date(now - timedelta(hours=24))
query = SentinelAPI.format_query(wkt, (last_24h, now))
assert query == 'beginPosition:[%s TO %s] ' % (last_24h, format_query_date(now)) + \
'footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"'
query = SentinelAPI.format_query(wkt,
date=(last_24h, "NOW"),
producttype='SLC',
raw='IW')
assert query == 'beginPosition:[%s TO NOW] ' % (format_query_date(last_24h)) + \
'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"'
query = SentinelAPI.format_query(wkt, producttype='SLC', raw='IW')
assert query == 'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"'
query = SentinelAPI.format_query(area=None, date=None)
assert query == ''
query = SentinelAPI.format_query()
assert query == ''
query = SentinelAPI.format_query(raw='test')
assert query == 'test'
def test_api_query_format():
wkt = "POLYGON((0 0,1 1,0 1,0 0))"
now = datetime.now()
last_24h = format_query_date(now - timedelta(hours=24))
query = SentinelAPI.format_query(wkt, (last_24h, now))
assert (query == "beginPosition:[%s TO %s] " %
(last_24h, format_query_date(now)) +
'footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"')
query = SentinelAPI.format_query(wkt,
date=(last_24h, "NOW"),
producttype="SLC",
raw="IW")
assert (
query == "beginPosition:[%s TO NOW] " % (format_query_date(last_24h)) +
'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"'
)
query = SentinelAPI.format_query(wkt, producttype="SLC", raw="IW")
assert query == 'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"'
query = SentinelAPI.format_query(area=None, date=None)
assert query == ""
query = SentinelAPI.format_query()
assert query == ""
query = SentinelAPI.format_query(raw="test")
assert query == "test"
def search(tile,
level='1C',
start='20150523',
end=datetime.datetime.today().strftime('%Y%m%d'),
maxcloud=100,
minsize=25.):
"""search(tile, start = '20161206', end = datetime.datetime.today().strftime('%Y%m%d'), maxcloud = 100, minsize_mb = 25.)
Searches for images from a single Sentinel-2 Granule that meet conditions of date range and cloud cover.
Args:
tile: A string containing the name of the tile to to download.
level: Download level '1C' (default) or '2A' data.
start: Start date for search in format YYYYMMDD. Defaults to 20150523.
end: End date for search in format YYYYMMDD. Defaults to today's date.
maxcloud: An integer of maximum percentage of cloud cover to download. Defaults to 100 %% (download all images, regardless of cloud cover).
minsize: A float with the minimum filesize to download in MB. Defaults to 25 MB. Be aware, file sizes smaller than this can result sen2three crashing.
Returns:
A pandas dataframe with details of scenes matching conditions.
"""
# Test that we're connected to the
assert 'scihub_api' in globals(
), "The global variable scihub_api doesn't exist. You should run connectToAPI(username, password) before searching the data archive."
# Validate tile input format for search
assert bool(
re.match("[0-9]{2}[A-Z]{3}$", tile)
), "The tile name input (%s) does not match the format ##XXX (e.g. 36KWA)." % tile
assert level in ['1C', '2A'], "Level must be '1C' or '2A'."
# Set up start and end dates
startdate = sentinelsat.format_query_date(start)
enddate = sentinelsat.format_query_date(end)
# Search data, filtering by options.
products = scihub_api.query(beginposition=(startdate, enddate),
platformname='Sentinel-2',
producttype='S2MSI%s' % level,
cloudcoverpercentage=(0, maxcloud),
filename='*T%s*' % tile)
# convert to Pandas DataFrame, which can be searched modified before commiting to download()
products_df = scihub_api.to_dataframe(products)
print('Found %s matching images' % str(len(products_df)))
# Where no results for tile
if len(products_df) == 0: return products_df
products_df['filesize_mb'] = _get_filesize(products_df)
products_df = products_df[products_df['filesize_mb'] >= float(minsize)]
return products_df
def search(search_area,
start='20140403',
end=datetime.datetime.today().strftime('%Y%m%d'),
direction='*'):
"""search(search_area, start = '20140403', end = datetime.datetime.today().strftime('%Y%m%d'), direction= '*')
Searches for Sentinel-1 GRD IW images that meet conditions of date range and extent.
Args:
search_area: A list in the format [minlon, minlat, maxlon, maxlat]
start: Start date for search in format YYYYMMDD. Start date may not precede 20140403, the launch date of Sentinel1-. Defaults to 20140403.
end: End date for search in format YYYYMMDD. Defaults to today's date.
Returns:
A pandas dataframe with details of scenes matching conditions.
"""
import sentinelsat
# Test that we're connected to SciHub
assert 'scihub_api' in globals(
), "The global variable scihub_api doesn't exist. You should run connectToAPI(username, password) before searching the data archive."
# Set up start and end dates
startdate = sentinelsat.format_query_date(start)
enddate = sentinelsat.format_query_date(end)
# Build a POLYGON wkt
search_polygon = _buildWkt(search_area)
# Search data, filtering by options.
products = scihub_api.query(search_polygon,
beginposition=(startdate, enddate),
platformname='Sentinel-1',
producttype='GRD',
orbitdirection=direction,
sensoroperationalmode='IW')
# convert to Pandas DataFrame, which can be searched modified before commiting to download()
products_df = scihub_api.to_dataframe(products)
if products_df.empty:
raise IOError("No products found. Check your search terms.")
return products_df
def test_format_date(api):
assert format_query_date(datetime(2015, 1, 1)) == '2015-01-01T00:00:00Z'
assert format_query_date(date(2015, 1, 1)) == '2015-01-01T00:00:00Z'
assert format_query_date('2015-01-01T00:00:00Z') == '2015-01-01T00:00:00Z'
assert format_query_date('20150101') == '2015-01-01T00:00:00Z'
assert format_query_date(' NOW ') == 'NOW'
assert format_query_date(None) == '*'
for date_str in ("NOW", "NOW-1DAY", "NOW-1DAYS", "NOW-500DAY", "NOW-500DAYS",
"NOW-2MONTH", "NOW-2MONTHS", "NOW-20MINUTE", "NOW-20MINUTES",
"NOW+10HOUR", "2015-01-01T00:00:00Z+1DAY", "NOW+3MONTHS-7DAYS/DAYS",
"*"):
assert format_query_date(date_str) == date_str
api.query(raw='ingestiondate:[{} TO *]'.format(date_str), limit=0)
for date_str in ("NOW - 1HOUR", "NOW - 1HOURS", "NOW-1 HOURS", "NOW-1", "NOW-", "**", "+", "-"):
with pytest.raises(ValueError):
format_query_date(date_str)
with pytest.raises(SentinelAPIError):
api.query(raw='ingestiondate:[{} TO *]'.format(date_str), limit=0)
def test_api_query_format():
wkt = 'POLYGON((0 0,1 1,0 1,0 0))'
now = datetime.now()
last_24h = format_query_date(now - timedelta(hours=24))
query = SentinelAPI.format_query(wkt, (last_24h, now))
assert query == 'beginPosition:[%s TO %s] ' % (last_24h, format_query_date(now)) + \
'footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"'
query = SentinelAPI.format_query(wkt, date=(last_24h, "NOW"), producttype='SLC', raw='IW')
assert query == 'beginPosition:[%s TO NOW] ' % (format_query_date(last_24h)) + \
'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"'
query = SentinelAPI.format_query(wkt, producttype='SLC', raw='IW')
assert query == 'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"'
query = SentinelAPI.format_query(area=None, date=None)
assert query == ''
query = SentinelAPI.format_query()
assert query == ''
query = SentinelAPI.format_query(raw='test')
assert query == 'test'
def test_format_date():
assert format_query_date(datetime(2015, 1, 1)) == '2015-01-01T00:00:00Z'
assert format_query_date(date(2015, 1, 1)) == '2015-01-01T00:00:00Z'
assert format_query_date('2015-01-01T00:00:00Z') == '2015-01-01T00:00:00Z'
assert format_query_date('20150101') == '2015-01-01T00:00:00Z'
assert format_query_date(' NOW ') == 'NOW'
assert format_query_date(None) == '*'
api = SentinelAPI(**_api_auth)
for date_str in ("NOW", "NOW-1DAY", "NOW-1DAYS", "NOW-500DAY", "NOW-500DAYS",
"NOW-2MONTH", "NOW-2MONTHS", "NOW-20MINUTE", "NOW-20MINUTES",
"NOW+10HOUR", "2015-01-01T00:00:00Z+1DAY", "NOW+3MONTHS-7DAYS/DAYS",
"*"):
assert format_query_date(date_str) == date_str
api.query(raw='ingestiondate:[{} TO *]'.format(date_str), limit=0)
for date_str in ("NOW - 1HOUR", "NOW - 1HOURS", "NOW-1 HOURS", "NOW-1", "NOW-", "**", "+", "-"):
with pytest.raises(ValueError):
format_query_date(date_str)
with pytest.raises(SentinelAPIError):
api.query(raw='ingestiondate:[{} TO *]'.format(date_str), limit=0)
def test_format_date():
assert format_query_date(datetime(2015, 1, 1)) == '2015-01-01T00:00:00Z'
assert format_query_date(date(2015, 1, 1)) == '2015-01-01T00:00:00Z'
assert format_query_date('2015-01-01T00:00:00Z') == '2015-01-01T00:00:00Z'
assert format_query_date('20150101') == '2015-01-01T00:00:00Z'
for date_str in ("NOW", "NOW-1DAY", "NOW-1DAYS", "NOW-500DAY", "NOW-500DAYS",
"NOW-2MONTH", "NOW-2MONTHS", "NOW-20MINUTE", "NOW-20MINUTES",
"NOW+10HOUR", "2015-01-01T00:00:00Z+1DAY", "NOW+3MONTHS-7DAYS/DAYS"):
assert format_query_date(date_str) == date_str
for date_str in ("NOW - 1HOUR", "NOW - 1HOURS", "NOW-1 HOURS", "NOW-1", "NOW-"):
with pytest.raises(ValueError) as excinfo:
format_query_date(date_str)
def test_format_date():
assert format_query_date(datetime(2015, 1, 1)) == '2015-01-01T00:00:00Z'
assert format_query_date(date(2015, 1, 1)) == '2015-01-01T00:00:00Z'
assert format_query_date('2015-01-01T00:00:00Z') == '2015-01-01T00:00:00Z'
assert format_query_date('20150101') == '2015-01-01T00:00:00Z'
for date_str in ("NOW", "NOW-1DAY", "NOW-1DAYS", "NOW-500DAY", "NOW-500DAYS",
"NOW-2MONTH", "NOW-2MONTHS", "NOW-20MINUTE", "NOW-20MINUTES",
"NOW+10HOUR", "2015-01-01T00:00:00Z+1DAY", "NOW+3MONTHS-7DAYS/DAYS"):
assert format_query_date(date_str) == date_str
for date_str in ("NOW - 1HOUR", "NOW - 1HOURS", "NOW-1 HOURS", "NOW-1", "NOW-"):
with pytest.raises(ValueError) as excinfo:
format_query_date(date_str)
def test_format_date(api):
assert format_query_date(datetime(2015, 1, 1)) == "2015-01-01T00:00:00Z"
assert format_query_date(date(2015, 1, 1)) == "2015-01-01T00:00:00Z"
assert format_query_date("2015-01-01T00:00:00Z") == "2015-01-01T00:00:00Z"
assert format_query_date("20150101") == "2015-01-01T00:00:00Z"
assert format_query_date(" NOW ") == "NOW"
assert format_query_date(None) == "*"
for date_str in (
"NOW",
"NOW-1DAY",
"NOW-1DAYS",
"NOW-500DAY",
"NOW-500DAYS",
"NOW-2MONTH",
"NOW-2MONTHS",
"NOW-20MINUTE",
"NOW-20MINUTES",
"NOW+10HOUR",
"2015-01-01T00:00:00Z+1DAY",
"NOW+3MONTHS-7DAYS/DAYS",
"*",
):
assert format_query_date(date_str) == date_str
api.query(raw="ingestiondate:[{} TO *]".format(date_str), limit=0)
for date_str in (
"NOW - 1HOUR",
"NOW - 1HOURS",
"NOW-1 HOURS",
"NOW-1",
"NOW-",
"**",
"+",
"-",
):
with pytest.raises(ValueError):
format_query_date(date_str)
with pytest.raises(QuerySyntaxError):
api.query(raw="ingestiondate:[{} TO *]".format(date_str), limit=0)
def query_metadata(self, platform, date, aoi, cloud_cover=None):
"""Queries satellite image metadata from data source.
:param platform: Image platform (<enum 'Platform'>).
:param date: Date from - to (String or Datetime tuple). Expects a tuple of (start, end), e.g.
(yyyyMMdd, yyyy-MM-ddThh:mm:ssZ, NOW, NOW-<n>DAY(S), HOUR(S), MONTH(S), etc.)
:param aoi: Area of interest as GeoJson file or bounding box tuple with lat lon coordinates (String, Tuple).
:param cloud_cover: Percent cloud cover scene from - to (Integer tuple).
:returns: Metadata of products that match query criteria (List of Metadata objects).
"""
if self.src == Datahub.File:
# query Filesystem for metadata
start_date = sentinelsat.format_query_date(date[0])
end_date = sentinelsat.format_query_date(date[1])
geom = self.prep_aoi(aoi)
if cloud_cover and platform != platform.Sentinel1:
min_cloud_cover = cloud_cover[0]
max_cloud_cover = cloud_cover[1]
else:
min_cloud_cover = 0
max_cloud_cover = 100
# get all json files in datadir that match substr
meta_files = sorted(
[
Path(dp).joinpath(f)
for dp, dn, filenames in os.walk(self.api)
for substr in self.api_substr
for f in filenames if f.endswith(".json") and substr in f
],
key=lambda path: str(path).lower(),
)
# filter json files by query parameters
meta_src = []
for meta_file in meta_files:
with open(meta_file) as f:
m = json.load(f)
try:
self.construct_metadata(meta_src=m, platform=platform)
except (json.decoder.JSONDecodeError, LookupError,
TypeError) as e:
raise ValueError(
f"{meta_file.name} not a valid metadata file. {e}."
)
m_platform = m["properties"]["platformname"]
m_date = sentinelsat.format_query_date(
m["properties"]["acquisitiondate"])
m_geom = geometry.shape(m["geometry"])
m_cloud_cover = m["properties"]["cloudcoverpercentage"]
if m_cloud_cover is None:
m_cloud_cover = 0
if (m_platform == platform.value
and start_date <= m_date < end_date
and m_geom.intersects(geom) and min_cloud_cover <=
m_cloud_cover < max_cloud_cover):
meta_src.append(m)
elif self.src == Datahub.EarthExplorer:
# query Earthexplorer for metadata
bbox = self.prep_aoi(aoi).bounds
kwargs = {}
if cloud_cover:
kwargs["max_cloud_cover"] = cloud_cover[1]
meta_src = self.api.search(
dataset=platform.value,
bbox=[bbox[1], bbox[0], bbox[3], bbox[2]],
start_date=sentinelsat.format_query_date(date[0]),
end_date=sentinelsat.format_query_date(date[1]),
max_results=10000,
**kwargs,
)
else:
# query Scihub for metadata
kwargs = {}
if cloud_cover and platform != platform.Sentinel1:
kwargs["cloudcoverpercentage"] = cloud_cover
meta_src = self.api.query(
area=self.prep_aoi(aoi).wkt,
date=date,
platformname=platform.value,
**kwargs,
)
meta_src = self.api.to_geojson(meta_src)["features"]
# construct MetadataCollection from list of Metadata objects
return MetadataCollection([
self.construct_metadata(meta_src=m, platform=platform)
for m in meta_src
])
def query_metadata(self,
platform,
date,
aoi,
cloud_cover=None,
kwargs=None):
"""Queries metadata from data source.
:param platform: Image platform (<enum 'Platform'>).
:param date: Date from - to in format yyyyMMdd (String or Datetime tuple).
:param aoi: Area of interest as GeoJson file or bounding box tuple with lat lon coordinates (String, Tuple).
:param cloud_cover: Percent cloud cover scene from - to (Integer tuple) (default: None).
:param kwargs: Dictionary of the additional requirements for the hub used (default: None).
:generates: Metadata item of products that match query criteria (PySTAC item).
"""
if kwargs is None:
kwargs = {}
if self.src == Datahub.STAC_local:
# query STAC Catalog for metadata
geom = self._prep_aoi(aoi)
for item in self.api.get_all_items():
if item.ext.eo.cloud_cover and cloud_cover:
if not cloud_cover[
0] <= item.ext.eo.cloud_cover < cloud_cover[1]:
continue
if (platform.value == item.common_metadata.platform
and sentinelsat.format_query_date(date[0]) <=
sentinelsat.format_query_date(
parse(item.properties["acquisitiondate"]).strftime(
"%Y%m%d")) < sentinelsat.format_query_date(
date[1])
and geometry.shape(item.geometry).intersects(geom)):
yield item
elif self.src == Datahub.STAC_API:
raise NotImplementedError(
f"Do this directly with our StacApi functionalities, see "
f"https://ukis-pysat.readthedocs.io/en/latest/api/stacapi.html."
)
elif self.src == Datahub.EarthExplorer:
# query EarthExplorer for metadata
bbox = self._prep_aoi(aoi).bounds
if cloud_cover:
kwargs["max_cloud_cover"] = cloud_cover[1]
products = self.api.search(
dataset=platform.value,
bbox=bbox,
start_date=sentinelsat.format_query_date(date[0]),
end_date=sentinelsat.format_query_date(date[1]),
max_results=10000,
**kwargs,
)
else:
# query Scihub for metadata
if cloud_cover and platform != platform.Sentinel1:
kwargs["cloudcoverpercentage"] = cloud_cover
products = self.api.query(
area=self._prep_aoi(aoi).wkt,
date=date,
platformname=platform.value,
**kwargs,
)
products = self.api.to_geojson(products)["features"]
for meta in products:
yield self.construct_metadata(meta=meta, platform=platform)
