def test_api_query_format(): wkt = 'POLYGON((0 0,1 1,0 1,0 0))' now = datetime.now() last_24h = format_query_date(now - timedelta(hours=24)) query = SentinelAPI.format_query(wkt, (last_24h, now)) assert query == 'beginPosition:[%s TO %s] ' % (last_24h, format_query_date(now)) + \ 'footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"' query = SentinelAPI.format_query(wkt, date=(last_24h, "NOW"), producttype='SLC', raw='IW') assert query == 'beginPosition:[%s TO NOW] ' % (format_query_date(last_24h)) + \ 'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"' query = SentinelAPI.format_query(wkt, producttype='SLC', raw='IW') assert query == 'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"' query = SentinelAPI.format_query(area=None, date=None) assert query == '' query = SentinelAPI.format_query() assert query == '' query = SentinelAPI.format_query(raw='test') assert query == 'test'
def test_api_query_format(): wkt = "POLYGON((0 0,1 1,0 1,0 0))" now = datetime.now() last_24h = format_query_date(now - timedelta(hours=24)) query = SentinelAPI.format_query(wkt, (last_24h, now)) assert (query == "beginPosition:[%s TO %s] " % (last_24h, format_query_date(now)) + 'footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"') query = SentinelAPI.format_query(wkt, date=(last_24h, "NOW"), producttype="SLC", raw="IW") assert ( query == "beginPosition:[%s TO NOW] " % (format_query_date(last_24h)) + 'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"' ) query = SentinelAPI.format_query(wkt, producttype="SLC", raw="IW") assert query == 'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"' query = SentinelAPI.format_query(area=None, date=None) assert query == "" query = SentinelAPI.format_query() assert query == "" query = SentinelAPI.format_query(raw="test") assert query == "test"
def search(tile, level='1C', start='20150523', end=datetime.datetime.today().strftime('%Y%m%d'), maxcloud=100, minsize=25.): """search(tile, start = '20161206', end = datetime.datetime.today().strftime('%Y%m%d'), maxcloud = 100, minsize_mb = 25.) Searches for images from a single Sentinel-2 Granule that meet conditions of date range and cloud cover. Args: tile: A string containing the name of the tile to to download. level: Download level '1C' (default) or '2A' data. start: Start date for search in format YYYYMMDD. Defaults to 20150523. end: End date for search in format YYYYMMDD. Defaults to today's date. maxcloud: An integer of maximum percentage of cloud cover to download. Defaults to 100 %% (download all images, regardless of cloud cover). minsize: A float with the minimum filesize to download in MB. Defaults to 25 MB. Be aware, file sizes smaller than this can result sen2three crashing. Returns: A pandas dataframe with details of scenes matching conditions. """ # Test that we're connected to the assert 'scihub_api' in globals( ), "The global variable scihub_api doesn't exist. You should run connectToAPI(username, password) before searching the data archive." # Validate tile input format for search assert bool( re.match("[0-9]{2}[A-Z]{3}$", tile) ), "The tile name input (%s) does not match the format ##XXX (e.g. 36KWA)." % tile assert level in ['1C', '2A'], "Level must be '1C' or '2A'." # Set up start and end dates startdate = sentinelsat.format_query_date(start) enddate = sentinelsat.format_query_date(end) # Search data, filtering by options. products = scihub_api.query(beginposition=(startdate, enddate), platformname='Sentinel-2', producttype='S2MSI%s' % level, cloudcoverpercentage=(0, maxcloud), filename='*T%s*' % tile) # convert to Pandas DataFrame, which can be searched modified before commiting to download() products_df = scihub_api.to_dataframe(products) print('Found %s matching images' % str(len(products_df))) # Where no results for tile if len(products_df) == 0: return products_df products_df['filesize_mb'] = _get_filesize(products_df) products_df = products_df[products_df['filesize_mb'] >= float(minsize)] return products_df
def search(search_area, start='20140403', end=datetime.datetime.today().strftime('%Y%m%d'), direction='*'): """search(search_area, start = '20140403', end = datetime.datetime.today().strftime('%Y%m%d'), direction= '*') Searches for Sentinel-1 GRD IW images that meet conditions of date range and extent. Args: search_area: A list in the format [minlon, minlat, maxlon, maxlat] start: Start date for search in format YYYYMMDD. Start date may not precede 20140403, the launch date of Sentinel1-. Defaults to 20140403. end: End date for search in format YYYYMMDD. Defaults to today's date. Returns: A pandas dataframe with details of scenes matching conditions. """ import sentinelsat # Test that we're connected to SciHub assert 'scihub_api' in globals( ), "The global variable scihub_api doesn't exist. You should run connectToAPI(username, password) before searching the data archive." # Set up start and end dates startdate = sentinelsat.format_query_date(start) enddate = sentinelsat.format_query_date(end) # Build a POLYGON wkt search_polygon = _buildWkt(search_area) # Search data, filtering by options. products = scihub_api.query(search_polygon, beginposition=(startdate, enddate), platformname='Sentinel-1', producttype='GRD', orbitdirection=direction, sensoroperationalmode='IW') # convert to Pandas DataFrame, which can be searched modified before commiting to download() products_df = scihub_api.to_dataframe(products) if products_df.empty: raise IOError("No products found. Check your search terms.") return products_df
def test_format_date(api): assert format_query_date(datetime(2015, 1, 1)) == '2015-01-01T00:00:00Z' assert format_query_date(date(2015, 1, 1)) == '2015-01-01T00:00:00Z' assert format_query_date('2015-01-01T00:00:00Z') == '2015-01-01T00:00:00Z' assert format_query_date('20150101') == '2015-01-01T00:00:00Z' assert format_query_date(' NOW ') == 'NOW' assert format_query_date(None) == '*' for date_str in ("NOW", "NOW-1DAY", "NOW-1DAYS", "NOW-500DAY", "NOW-500DAYS", "NOW-2MONTH", "NOW-2MONTHS", "NOW-20MINUTE", "NOW-20MINUTES", "NOW+10HOUR", "2015-01-01T00:00:00Z+1DAY", "NOW+3MONTHS-7DAYS/DAYS", "*"): assert format_query_date(date_str) == date_str api.query(raw='ingestiondate:[{} TO *]'.format(date_str), limit=0) for date_str in ("NOW - 1HOUR", "NOW - 1HOURS", "NOW-1 HOURS", "NOW-1", "NOW-", "**", "+", "-"): with pytest.raises(ValueError): format_query_date(date_str) with pytest.raises(SentinelAPIError): api.query(raw='ingestiondate:[{} TO *]'.format(date_str), limit=0)
def test_api_query_format(): wkt = 'POLYGON((0 0,1 1,0 1,0 0))' now = datetime.now() last_24h = format_query_date(now - timedelta(hours=24)) query = SentinelAPI.format_query(wkt, (last_24h, now)) assert query == 'beginPosition:[%s TO %s] ' % (last_24h, format_query_date(now)) + \ 'footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"' query = SentinelAPI.format_query(wkt, date=(last_24h, "NOW"), producttype='SLC', raw='IW') assert query == 'beginPosition:[%s TO NOW] ' % (format_query_date(last_24h)) + \ 'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"' query = SentinelAPI.format_query(wkt, producttype='SLC', raw='IW') assert query == 'producttype:SLC IW footprint:"Intersects(POLYGON((0 0,1 1,0 1,0 0)))"' query = SentinelAPI.format_query(area=None, date=None) assert query == '' query = SentinelAPI.format_query() assert query == '' query = SentinelAPI.format_query(raw='test') assert query == 'test'
def test_format_date(): assert format_query_date(datetime(2015, 1, 1)) == '2015-01-01T00:00:00Z' assert format_query_date(date(2015, 1, 1)) == '2015-01-01T00:00:00Z' assert format_query_date('2015-01-01T00:00:00Z') == '2015-01-01T00:00:00Z' assert format_query_date('20150101') == '2015-01-01T00:00:00Z' assert format_query_date(' NOW ') == 'NOW' assert format_query_date(None) == '*' api = SentinelAPI(**_api_auth) for date_str in ("NOW", "NOW-1DAY", "NOW-1DAYS", "NOW-500DAY", "NOW-500DAYS", "NOW-2MONTH", "NOW-2MONTHS", "NOW-20MINUTE", "NOW-20MINUTES", "NOW+10HOUR", "2015-01-01T00:00:00Z+1DAY", "NOW+3MONTHS-7DAYS/DAYS", "*"): assert format_query_date(date_str) == date_str api.query(raw='ingestiondate:[{} TO *]'.format(date_str), limit=0) for date_str in ("NOW - 1HOUR", "NOW - 1HOURS", "NOW-1 HOURS", "NOW-1", "NOW-", "**", "+", "-"): with pytest.raises(ValueError): format_query_date(date_str) with pytest.raises(SentinelAPIError): api.query(raw='ingestiondate:[{} TO *]'.format(date_str), limit=0)
def test_format_date(): assert format_query_date(datetime(2015, 1, 1)) == '2015-01-01T00:00:00Z' assert format_query_date(date(2015, 1, 1)) == '2015-01-01T00:00:00Z' assert format_query_date('2015-01-01T00:00:00Z') == '2015-01-01T00:00:00Z' assert format_query_date('20150101') == '2015-01-01T00:00:00Z' for date_str in ("NOW", "NOW-1DAY", "NOW-1DAYS", "NOW-500DAY", "NOW-500DAYS", "NOW-2MONTH", "NOW-2MONTHS", "NOW-20MINUTE", "NOW-20MINUTES", "NOW+10HOUR", "2015-01-01T00:00:00Z+1DAY", "NOW+3MONTHS-7DAYS/DAYS"): assert format_query_date(date_str) == date_str for date_str in ("NOW - 1HOUR", "NOW - 1HOURS", "NOW-1 HOURS", "NOW-1", "NOW-"): with pytest.raises(ValueError) as excinfo: format_query_date(date_str)
def test_format_date(): assert format_query_date(datetime(2015, 1, 1)) == '2015-01-01T00:00:00Z' assert format_query_date(date(2015, 1, 1)) == '2015-01-01T00:00:00Z' assert format_query_date('2015-01-01T00:00:00Z') == '2015-01-01T00:00:00Z' assert format_query_date('20150101') == '2015-01-01T00:00:00Z' for date_str in ("NOW", "NOW-1DAY", "NOW-1DAYS", "NOW-500DAY", "NOW-500DAYS", "NOW-2MONTH", "NOW-2MONTHS", "NOW-20MINUTE", "NOW-20MINUTES", "NOW+10HOUR", "2015-01-01T00:00:00Z+1DAY", "NOW+3MONTHS-7DAYS/DAYS"): assert format_query_date(date_str) == date_str for date_str in ("NOW - 1HOUR", "NOW - 1HOURS", "NOW-1 HOURS", "NOW-1", "NOW-"): with pytest.raises(ValueError) as excinfo: format_query_date(date_str)
def test_format_date(api): assert format_query_date(datetime(2015, 1, 1)) == "2015-01-01T00:00:00Z" assert format_query_date(date(2015, 1, 1)) == "2015-01-01T00:00:00Z" assert format_query_date("2015-01-01T00:00:00Z") == "2015-01-01T00:00:00Z" assert format_query_date("20150101") == "2015-01-01T00:00:00Z" assert format_query_date(" NOW ") == "NOW" assert format_query_date(None) == "*" for date_str in ( "NOW", "NOW-1DAY", "NOW-1DAYS", "NOW-500DAY", "NOW-500DAYS", "NOW-2MONTH", "NOW-2MONTHS", "NOW-20MINUTE", "NOW-20MINUTES", "NOW+10HOUR", "2015-01-01T00:00:00Z+1DAY", "NOW+3MONTHS-7DAYS/DAYS", "*", ): assert format_query_date(date_str) == date_str api.query(raw="ingestiondate:[{} TO *]".format(date_str), limit=0) for date_str in ( "NOW - 1HOUR", "NOW - 1HOURS", "NOW-1 HOURS", "NOW-1", "NOW-", "**", "+", "-", ): with pytest.raises(ValueError): format_query_date(date_str) with pytest.raises(QuerySyntaxError): api.query(raw="ingestiondate:[{} TO *]".format(date_str), limit=0)
def query_metadata(self, platform, date, aoi, cloud_cover=None): """Queries satellite image metadata from data source. :param platform: Image platform (<enum 'Platform'>). :param date: Date from - to (String or Datetime tuple). Expects a tuple of (start, end), e.g. (yyyyMMdd, yyyy-MM-ddThh:mm:ssZ, NOW, NOW-<n>DAY(S), HOUR(S), MONTH(S), etc.) :param aoi: Area of interest as GeoJson file or bounding box tuple with lat lon coordinates (String, Tuple). :param cloud_cover: Percent cloud cover scene from - to (Integer tuple). :returns: Metadata of products that match query criteria (List of Metadata objects). """ if self.src == Datahub.File: # query Filesystem for metadata start_date = sentinelsat.format_query_date(date[0]) end_date = sentinelsat.format_query_date(date[1]) geom = self.prep_aoi(aoi) if cloud_cover and platform != platform.Sentinel1: min_cloud_cover = cloud_cover[0] max_cloud_cover = cloud_cover[1] else: min_cloud_cover = 0 max_cloud_cover = 100 # get all json files in datadir that match substr meta_files = sorted( [ Path(dp).joinpath(f) for dp, dn, filenames in os.walk(self.api) for substr in self.api_substr for f in filenames if f.endswith(".json") and substr in f ], key=lambda path: str(path).lower(), ) # filter json files by query parameters meta_src = [] for meta_file in meta_files: with open(meta_file) as f: m = json.load(f) try: self.construct_metadata(meta_src=m, platform=platform) except (json.decoder.JSONDecodeError, LookupError, TypeError) as e: raise ValueError( f"{meta_file.name} not a valid metadata file. {e}." ) m_platform = m["properties"]["platformname"] m_date = sentinelsat.format_query_date( m["properties"]["acquisitiondate"]) m_geom = geometry.shape(m["geometry"]) m_cloud_cover = m["properties"]["cloudcoverpercentage"] if m_cloud_cover is None: m_cloud_cover = 0 if (m_platform == platform.value and start_date <= m_date < end_date and m_geom.intersects(geom) and min_cloud_cover <= m_cloud_cover < max_cloud_cover): meta_src.append(m) elif self.src == Datahub.EarthExplorer: # query Earthexplorer for metadata bbox = self.prep_aoi(aoi).bounds kwargs = {} if cloud_cover: kwargs["max_cloud_cover"] = cloud_cover[1] meta_src = self.api.search( dataset=platform.value, bbox=[bbox[1], bbox[0], bbox[3], bbox[2]], start_date=sentinelsat.format_query_date(date[0]), end_date=sentinelsat.format_query_date(date[1]), max_results=10000, **kwargs, ) else: # query Scihub for metadata kwargs = {} if cloud_cover and platform != platform.Sentinel1: kwargs["cloudcoverpercentage"] = cloud_cover meta_src = self.api.query( area=self.prep_aoi(aoi).wkt, date=date, platformname=platform.value, **kwargs, ) meta_src = self.api.to_geojson(meta_src)["features"] # construct MetadataCollection from list of Metadata objects return MetadataCollection([ self.construct_metadata(meta_src=m, platform=platform) for m in meta_src ])
def query_metadata(self, platform, date, aoi, cloud_cover=None, kwargs=None): """Queries metadata from data source. :param platform: Image platform (<enum 'Platform'>). :param date: Date from - to in format yyyyMMdd (String or Datetime tuple). :param aoi: Area of interest as GeoJson file or bounding box tuple with lat lon coordinates (String, Tuple). :param cloud_cover: Percent cloud cover scene from - to (Integer tuple) (default: None). :param kwargs: Dictionary of the additional requirements for the hub used (default: None). :generates: Metadata item of products that match query criteria (PySTAC item). """ if kwargs is None: kwargs = {} if self.src == Datahub.STAC_local: # query STAC Catalog for metadata geom = self._prep_aoi(aoi) for item in self.api.get_all_items(): if item.ext.eo.cloud_cover and cloud_cover: if not cloud_cover[ 0] <= item.ext.eo.cloud_cover < cloud_cover[1]: continue if (platform.value == item.common_metadata.platform and sentinelsat.format_query_date(date[0]) <= sentinelsat.format_query_date( parse(item.properties["acquisitiondate"]).strftime( "%Y%m%d")) < sentinelsat.format_query_date( date[1]) and geometry.shape(item.geometry).intersects(geom)): yield item elif self.src == Datahub.STAC_API: raise NotImplementedError( f"Do this directly with our StacApi functionalities, see " f"https://ukis-pysat.readthedocs.io/en/latest/api/stacapi.html." ) elif self.src == Datahub.EarthExplorer: # query EarthExplorer for metadata bbox = self._prep_aoi(aoi).bounds if cloud_cover: kwargs["max_cloud_cover"] = cloud_cover[1] products = self.api.search( dataset=platform.value, bbox=bbox, start_date=sentinelsat.format_query_date(date[0]), end_date=sentinelsat.format_query_date(date[1]), max_results=10000, **kwargs, ) else: # query Scihub for metadata if cloud_cover and platform != platform.Sentinel1: kwargs["cloudcoverpercentage"] = cloud_cover products = self.api.query( area=self._prep_aoi(aoi).wkt, date=date, platformname=platform.value, **kwargs, ) products = self.api.to_geojson(products)["features"] for meta in products: yield self.construct_metadata(meta=meta, platform=platform)