def __init__(self,
username,
password,
area_geom,
exclude=None,
**query_kw):
api = sentinelsat.SentinelAPI(user=username,
password=password,
show_progressbars=False)
results = query.query_s2(api, area_geom, **query_kw)
self._length = len(results)
exclude = set(exclude or [])
def _get_and_download_file(base_url):
product_url = find.get_tci_url(base_url, session=api.session)
if product_url is None:
return None, None
fname = download.fname_from_url(product_url)
if fname in exclude:
return None, fname
return download.stream_file(product_url,
session=api.session), fname
def _data_generator():
with executor.MaxSizeThreadPoolExecutor(queue_size=4,
max_workers=1) as exe:
with closing(api.session):
yield from exe.map(_get_and_download_file,
results.values())
self._generator = _data_generator
def download_unzip_transform_to_geotiff(product):
api = sentinelsat.SentinelAPI(user=os.environ["SENTINEL_USERNAME"],
password=os.environ["SENTINEL_PASSWORD"])
cos = COS(ibm_api_key_id=os.environ['IBM_API_KEY_ID'],
ibm_service_instance_id=os.environ['IBM_SERVICE_INSTANCE_ID'],
endpoint_url=os.environ['ENDPOINT_URL'],
bucket=os.environ['BUCKET'])
with tempfile.TemporaryDirectory() as tmpdir:
# Download a Sentinel-2 tile
d_meta = api.download(product['uuid'], directory_path=tmpdir)
# Extract and remove zip file
zip_ref = zipfile.ZipFile(d_meta['path'])
zip_ref.extractall(tmpdir)
zip_ref.close()
os.remove(d_meta['path'])
# Atmospheric correction
sentinel_product_dir = product['filename']
val = subprocess.check_call(
f'{os.environ["SEN2COR_COM"]} --resolution 10 {sentinel_product_dir}'
)
# Translate bands in .jp2 to Cloud Optimized geoTiff format
band4 = glob.glob(
f"{sentinel_product_dir[0:26].replace('1C', '2A')}*/GRANULE/*/IMG_DATA/R10m/*B04*"
)
band4 = band4[0] if len(band4) == 1 else None
band8 = glob.glob(
f"{sentinel_product_dir[0:26].replace('1C', '2A')}*/GRANULE/*/IMG_DATA/R10m/*B08*"
)
band8 = band8[0] if len(band8) == 1 else None
band4_tiff_file = jp2_to_cog(band4)
band8_tiff_file = jp2_to_cog(band8)
cos.upload_band_file(band4_tiff_file, product)
cos.upload_band_file(band8_tiff_file, product)
def downloadS5pData(startDate,
endDate,
footprint,
targetDir='./downloads/',
output='default'):
"""
S5p is available under the same API as other S1-3, but under a different url and credentials.
Instead of https://scihub.copernicus.eu/dhus/ use https://s5phub.copernicus.eu/dhus/,
and instead of your credentials use s5pguest/s5pguest.
"""
api = ss.SentinelAPI('s5pguest', 's5pguest',
'https://s5phub.copernicus.eu/dhus/')
products = api.query(
area=footprint,
area_relation='Intersects',
date=(startDate, endDate),
producttype='L2__NO2___', #L2__CH4___
platformname='Sentinel-5',
processinglevel='L2')
api.download_all(products, targetDir)
if output == 'default':
return api, products
if output == 'geopandas':
return api, api.to_geodataframe(products)
if output == 'pandas':
return api, api.to_dataframe(products)
if output == 'geojson':
return api, api.to_geojson(products)
def download_tci(username, password, area_geom, outdir, **query_kw):
api = sentinelsat.SentinelAPI(user=username, password=password)
session = api.session
logger.info('Querying SciHub')
results = query.query_s2(api, area_geom, **query_kw)
logger.info('Found %d products', len(results))
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as exe:
results_iter = tqdm.tqdm(results.values(),
desc='Retrieving TCI URLs',
unit='result')
urls = exe.map(functools.partial(find.get_tci_url, session=session),
results_iter)
urls = [url for url in urls if url is not None]
logger.info('Retrieved %d TCI download URLs', len(urls))
url_iter = tqdm.tqdm(urls, desc='Downloading TCI files', unit='file')
targets = exe.map(
functools.partial(download.download_file,
outdir=outdir,
session=session), url_iter)
logger.info('Downloaded %d files', len(targets))
return targets
def __init__(self, datahub, catalog=None, url=None):
"""
:param datahub: Data source (<enum 'Datahub'>).
:param catalog: Only applicable if datahub is 'STAC_local'. Can be one of the following types:
Path to STAC Catalog file catalog.json (String, Path).
Pystac Catalog or Collection object (pystac.catalog.Catalog, pystac.collection.Collection).
None initializes an empty catalog.
(default: None)
:param url: Only applicable if datahub is 'STAC_API'. STAC Server endpoint, reads from STAC_API_URL environment
variable by default
(default: None)
"""
self.src = datahub
if self.src == Datahub.STAC_local:
# connect to STAC Catalog
if isinstance(
catalog,
(pystac.catalog.Catalog, pystac.collection.Collection)):
self.api = catalog
elif isinstance(catalog, (str, Path)):
href = Path(catalog).resolve().as_uri()
self.api = pystac.catalog.Catalog.from_file(href)
elif catalog is None:
self.api = self._init_catalog()
else:
raise AttributeError(
f"{catalog} is not a valid STAC Catalog [catalog.json, pystac.catalog.Catalog, "
f"pystac.collection.Collection, None] ")
elif self.src == Datahub.STAC_API:
if url:
self.api = StacApi(url=url)
else:
self.api = StacApi()
elif self.src == Datahub.EarthExplorer:
import landsatxplore.api
# connect to Earthexplorer
self.user = env_get("EARTHEXPLORER_USER")
self.pw = env_get("EARTHEXPLORER_PW")
self.api = landsatxplore.api.API(self.user, self.pw)
elif self.src == Datahub.Scihub:
# connect to Scihub
self.user = env_get("SCIHUB_USER")
self.pw = env_get("SCIHUB_PW")
self.api = sentinelsat.SentinelAPI(
self.user,
self.pw,
"https://apihub.copernicus.eu/apihub",
show_progressbars=False,
)
else:
raise NotImplementedError(
f"{datahub} is not supported [STAC_local, STAC_API, EarthExplorer, Scihub]"
)
def get_parsed_granule_meta(auth, tile_name):
api = sentinelsat.SentinelAPI(*auth)
results = api.query(date=('NOW-5DAYS', 'NOW'),
tileid=tile_name,
platformname="Sentinel-2",
producttype="S2MSI1C")
r = next(iter(results.values()))
metas = get_sentinel_metas(r, auth=auth)
gm = s2meta.parse_granule_metadata(metadatastr=metas['granule_meta'][0])
return gm
def getlogin():
print "\nThis script requires a Copernicus Open Access Hub user ID and password."
print "Please enter your credentials here.\n"
print "If you don't have a login, you can register at https://scihub.copernicus.eu/\n"
print "\nWarning: Some python interpreters will echo password input or fail after username input."
print "To avoid this, run api_download.py from Command Prompt or Powershell.\n"
coah_user = raw_input("User Name:")
coah_pass = getpass.getpass("Password:")
login = sentinelsat.SentinelAPI(coah_user, coah_pass,
'https://scihub.copernicus.eu/dhus')
return login
def connectToAPI(username, password):
'''
Connect to the SciHub API with sentinelsat. Sign up at https://scihub.copernicus.eu/.
Args:
username: Scihub username.
password: Scihub password.
'''
# Connect to Sentinel API
return sentinelsat.SentinelAPI(username, password,
'https://scihub.copernicus.eu/dhus')
def download_from_sentinel(product, tmpdir):
api = sentinelsat.SentinelAPI(user=os.environ["SENTINEL_USERNAME"],
password=os.environ["SENTINEL_PASSWORD"])
sentinel_product_dir = product['filename']
if not os.path.exists(sentinel_product_dir):
d_meta = api.download(product['uuid'], directory_path=tmpdir)
# Extract and remove zip file
zip_ref = zipfile.ZipFile(d_meta['path'])
zip_ref.extractall(tmpdir)
zip_ref.close()
os.remove(d_meta['path'])
return sentinel_product_dir
def get_sentinel_metadata_from_area(from_date,
to_date,
geo_json_area,
cloudcoverpercentage=(0, 15)):
api = sentinelsat.SentinelAPI(user=os.environ["SENTINEL_USERNAME"],
password=os.environ["SENTINEL_PASSWORD"])
footprint = sentinelsat.geojson_to_wkt(geo_json_area)
products = api.query(footprint,
date=(from_date, to_date),
platformname='Sentinel-2',
producttype=('S2MS2Ap', 'S2MSI1C'),
cloudcoverpercentage=cloudcoverpercentage)
return products
def connectToAPI(username, password):
'''
Connect to the SciHub API with sentinelsat. Sign up at https://scihub.copernicus.eu/.
Args:
username: Scihub username.
password: Scihub password.
'''
import sentinelsat
# Let API be accessed by other functions
global scihub_api
# Connect to Sentinel API
scihub_api = sentinelsat.SentinelAPI(username, password,
'https://scihub.copernicus.eu/dhus')
def connectToAPI(username, password):
'''
Connect to the SciHub API with sentinelsat.
Args:
username: Scihub username. Sign up at https://scihub.copernicus.eu/.
password: Scihub password.
'''
# Let API be accessed by other functions
global scihub_api
# Disconnect from any previous session
scihub_api = None
# Connect to Sentinel API
scihub_api = sentinelsat.SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
def __init__(self, datahub, datadir=None, datadir_substr=None):
"""
:param datahub: Data source (<enum 'Datahub'>).
:param datadir: Path to directory that holds the metadata if datahub is 'File' (String).
:param datadir_substr: Optional substring patterns to identify metadata in datadir if datahub
is 'File' (List of String).
"""
self.src = datahub
if self.src == Datahub.File:
if not datadir:
raise AttributeError(
f"'datadir' has to be set if datahub is 'File'.")
else:
self.api = datadir
if datadir_substr is None:
self.api_substr = [""]
else:
self.api_substr = datadir_substr
elif self.src == Datahub.EarthExplorer:
# connect to Earthexplorer
self.user = env_get("EARTHEXPLORER_USER")
self.pw = env_get("EARTHEXPLORER_PW")
self.api = landsatxplore.api.API(self.user, self.pw)
elif self.src == Datahub.Scihub:
# connect to Scihub
self.user = env_get("SCIHUB_USER")
self.pw = env_get("SCIHUB_PW")
self.api = sentinelsat.SentinelAPI(
self.user,
self.pw,
"https://scihub.copernicus.eu/dhus",
show_progressbars=False,
)
else:
raise NotImplementedError(
f"{datahub} is not supported [File, EarthExplorer, Scihub]")
def download_products(tiles,
start_date,
end_date,
output_folder,
show_progressbars=True):
"""
Descarga todos los productos del satélite Sentinel-2 para los tipos de producto S2MS2Ap y S2MSI1C
:param tiles: Tiles para filtrar la descarga
:param start_date: Fecha inicial en que se tomaron las imágenes
:param end_date: Fecha final en que se tomaron las imágenes
:param output_folder: Directorio en el que se almacenarán las imágenes
:param show_progressbars: Indica si se muestran las barras de progreso durante la descarga
"""
print('Downloading products')
api = sentinelsat.SentinelAPI(user=SENT_USER,
password=SENT_PASS,
api_url='https://scihub.copernicus.eu/dhus',
show_progressbars=show_progressbars)
query_kwargs = {
'platformname': 'Sentinel-2',
'producttype': ('S2MS2Ap', 'S2MSI1C'),
'cloudcoverpercentage': (0, 15),
'date': (start_date, end_date)
}
products = collections.OrderedDict()
for tile in tiles:
kw = query_kwargs.copy()
kw['tileid'] = tile
pp = api.query(**kw)
products.update(pp)
api.download_all(products, output_folder)
def download_products(tiles,
start_date,
end_date,
output_folder,
show_progressbars=True):
"""
Download all Sentinel-2 satellite products for product types S2MS2Ap and S2MSI1C
:param tiles: Tiles to filter the download
:param start_date: Starting date images were taken
:param end_date: Final date images were taken
:param output_folder: Directory in which the images will be stored
:param show_progressbars: Indicates if progress bars are displayed during download
"""
print('Downloading products')
api = sentinelsat.SentinelAPI(user=SENT_USER,
password=SENT_PASS,
api_url='https://scihub.copernicus.eu/dhus',
show_progressbars=show_progressbars)
query_kwargs = {
'platformname': 'Sentinel-2',
'producttype': ('S2MS2Ap', 'S2MSI1C'),
'cloudcoverpercentage': (0, 15),
'date': (start_date, end_date)
}
products = collections.OrderedDict()
for tile in tiles:
kw = query_kwargs.copy()
kw['tileid'] = tile
pp = api.query(**kw)
products.update(pp)
api.download_all(products, output_folder)
def get_geojson_info(products):
api = sentinelsat.SentinelAPI(user=os.environ["SENTINEL_USERNAME"],
password=os.environ["SENTINEL_PASSWORD"])
return api.to_geojson(products)
def get_remote_files_eumetsat(**kwargs):
'''
Download swath files from EUMETSAT and store them at defined
location. This fct uses the SentinelAPI for queries.
'''
product = kwargs.get('product')
sdate = kwargs.get('sdate')
edate = kwargs.get('edate')
mission = kwargs.get('mission','s3a')
path_local = kwargs.get('path_local')
dict_for_sub = kwargs.get('dict_for_sub')
api_url = kwargs.get('api_url')
import sentinelsat as ss
products = None
dates = (sdate.strftime('%Y-%m-%dT%H:%M:%SZ'),\
edate.strftime('%Y-%m-%dT%H:%M:%SZ'))
filesort = False
if path_local is None:
# create local path
path_template = satellite_dict[product]['dst']\
['path_template']
strsublst = satellite_dict[product]['dst']\
['strsub']
subdict = make_subdict(strsublst,
class_object_dict=dict_for_sub)
path_local = make_pathtofile(path_template,\
strsublst,
subdict,\
date=sdate)
filesort = True
query_dict = make_query_dict(product,mission)
print(query_dict)
if api_url is None:
api_url_lst = \
satellite_dict[product]['src']['api_url']
for url in api_url_lst:
print('Source:',url)
try:
user, pw = get_credentials(remoteHostName=url)
api = ss.SentinelAPI(user, pw, url)
products = api.query(area=None, date=dates,**query_dict)
break
except Exception as e:
print(e)
else:
user, pw = get_credentials(remoteHostName = api_url)
api = ss.SentinelAPI(user, pw, api_url)
products = api.query(area=None, date=dates,**query_dict)
if products is not None:
# check if download path exists if not create
if not os.path.exists(path_local):
os.makedirs(path_local,exist_ok=True)
api.download_all(products,directory_path=path_local)
#api.download(product_id)
else: print('No products found!')
if filesort is True:
# sort files
print("Data is being sorted into subdirectories " \
+ "year and month ...")
filelst = [f for f in os.listdir(path_local)
if os.path.isfile(os.path.join(path_local,f))]
sort_files(path_local,filelst,product,mission)
print ('Files downloaded to: \n', path_local)
"""
Script to download the Sentinel-2 images from the long time archive,
for the extent of the national park
"""
import sentinelsat
from datetime import date
import logging
import tkinter
# login to scihub
api = sentinelsat.SentinelAPI('johannes0592', 'QthBa9z7x!')
nationalparc = r"F:\Studium_Trier\Masterarbeit\Datensaetze\Geojson\NationalparkGrenzenVereinfacht.json"
#specify boundaries to select images
footprint = sentinelsat.geojson_to_wkt(sentinelsat.read_geojson(nationalparc))
products = api.query(footprint,
date=(date(2019, 10, 1), date(2020, 9, 23)),
platformname='Sentinel-2',
cloudcoverpercentage=(0, 60),
processinglevel='Level-1C',
tileid='32ULA')
ids = []
x = products.keys()
for item in x:
ids.append(item)
# start loop every 30 minutes again to order new images
def downloadLTA():
# Create ring
ring = ogr.Geometry(ogr.wkbLinearRing)
ring.AddPoint(minX, minY)
ring.AddPoint(maxX, minY)
ring.AddPoint(maxX, maxY)
ring.AddPoint(minX, maxY)
ring.AddPoint(minX, minY)
# Create polygon
poly_envelope = ogr.Geometry(ogr.wkbPolygon)
poly_envelope.AddGeometry(ring)
return poly_envelope
# Login info to the sentinel2 repository
api = sentinelsat.SentinelAPI('runaas', 'FineBilder',
'https://scihub.copernicus.eu/dhus')
# Login to the database
with psycopg2.connect(
"host=beistet port=5433 dbname=LDir user=postgres password=1234"
) as conn:
with conn.cursor("geo_cur") as cur:
# Do reqest
# This should be narrowed down, currently it retrieves the whole database...
cur.execute("""SELECT ST_AsBinary(geog), dp_fra, dp_til FROM pa""")
dp_fra = dt.date.today()
dp_til = dt.date.min
geo_collection = ogr.Geometry(ogr.wkbMultiPolygon)
for row in cur:
geo_frm_db = ogr.CreateGeometryFromWkb(row[0])
if row[1] and row[1] < dp_fra:
import sentinelsat
import collections
# https://sentinelsat.readthedocs.io/en/stable/api.html
api = sentinelsat.SentinelAPI(user="******",
password="******",
api_url='https://scihub.copernicus.eu/dhus',
show_progressbars=True)
# Reading all the tiles
tiles = []
tiletxt = open("SpainPen_tiles.txt", mode="r")
tilelines = tiletxt.read().splitlines()
tiletxt.close()
for t in tilelines:
tiles.append(t)
# tiles = ["29SPB"]
#specify start and end date (here we use May 2017)
dateStart = "20180101"
dateEnd = "20181231"
query_kwargs = {
'platformname': 'Sentinel-2',
'producttype': ('S2MS2Ap', 'S2MSI1C'),
'cloudcoverpercentage': (0, 15),
'date': (dateStart, dateEnd)
}
import sentinelsat
from datetime import date
import pandas as pd
import time
### Set your credentials
finhub_user = '******'
finhub_pwd = 'a'
finhub_url = 'https://finhub.nsdc.fmi.fi'
scihub_user = '******'
scihub_pwd = 'a'
scihub_url = 'https://scihub.copernicus.eu/dhus'
### Open API connection
finhub_api = sentinelsat.SentinelAPI(finhub_user, finhub_pwd, finhub_url)
scihub_api = sentinelsat.SentinelAPI(scihub_user, scihub_pwd, scihub_url)
### Search by polygon (WGS84), time, and query keywords
footprint = sentinelsat.geojson_to_wkt(
sentinelsat.read_geojson(r'helsinki.geojson'))
startDate = date(2020, 1, 1)
endDate = date(2020, 7, 30)
cloudcoverage = (0, 20)
platformname = 'Sentinel-2'
producttype = 'S2MSI1C'
# producttype='S2MSI2A' #for L2 images
area_relation = "Contains" # footprint has to be fully inside the image. Other options "Intersects", "IsWithin"
### If your area is between two UTM zones, this script often downloads two versions of the same image
### Uncomment and add e.g "T35" to only focus on one UTM zone
