def __init__(self, username, password, area_geom, exclude=None, **query_kw): api = sentinelsat.SentinelAPI(user=username, password=password, show_progressbars=False) results = query.query_s2(api, area_geom, **query_kw) self._length = len(results) exclude = set(exclude or []) def _get_and_download_file(base_url): product_url = find.get_tci_url(base_url, session=api.session) if product_url is None: return None, None fname = download.fname_from_url(product_url) if fname in exclude: return None, fname return download.stream_file(product_url, session=api.session), fname def _data_generator(): with executor.MaxSizeThreadPoolExecutor(queue_size=4, max_workers=1) as exe: with closing(api.session): yield from exe.map(_get_and_download_file, results.values()) self._generator = _data_generator
def download_unzip_transform_to_geotiff(product): api = sentinelsat.SentinelAPI(user=os.environ["SENTINEL_USERNAME"], password=os.environ["SENTINEL_PASSWORD"]) cos = COS(ibm_api_key_id=os.environ['IBM_API_KEY_ID'], ibm_service_instance_id=os.environ['IBM_SERVICE_INSTANCE_ID'], endpoint_url=os.environ['ENDPOINT_URL'], bucket=os.environ['BUCKET']) with tempfile.TemporaryDirectory() as tmpdir: # Download a Sentinel-2 tile d_meta = api.download(product['uuid'], directory_path=tmpdir) # Extract and remove zip file zip_ref = zipfile.ZipFile(d_meta['path']) zip_ref.extractall(tmpdir) zip_ref.close() os.remove(d_meta['path']) # Atmospheric correction sentinel_product_dir = product['filename'] val = subprocess.check_call( f'{os.environ["SEN2COR_COM"]} --resolution 10 {sentinel_product_dir}' ) # Translate bands in .jp2 to Cloud Optimized geoTiff format band4 = glob.glob( f"{sentinel_product_dir[0:26].replace('1C', '2A')}*/GRANULE/*/IMG_DATA/R10m/*B04*" ) band4 = band4[0] if len(band4) == 1 else None band8 = glob.glob( f"{sentinel_product_dir[0:26].replace('1C', '2A')}*/GRANULE/*/IMG_DATA/R10m/*B08*" ) band8 = band8[0] if len(band8) == 1 else None band4_tiff_file = jp2_to_cog(band4) band8_tiff_file = jp2_to_cog(band8) cos.upload_band_file(band4_tiff_file, product) cos.upload_band_file(band8_tiff_file, product)
def downloadS5pData(startDate, endDate, footprint, targetDir='./downloads/', output='default'): """ S5p is available under the same API as other S1-3, but under a different url and credentials. Instead of https://scihub.copernicus.eu/dhus/ use https://s5phub.copernicus.eu/dhus/, and instead of your credentials use s5pguest/s5pguest. """ api = ss.SentinelAPI('s5pguest', 's5pguest', 'https://s5phub.copernicus.eu/dhus/') products = api.query( area=footprint, area_relation='Intersects', date=(startDate, endDate), producttype='L2__NO2___', #L2__CH4___ platformname='Sentinel-5', processinglevel='L2') api.download_all(products, targetDir) if output == 'default': return api, products if output == 'geopandas': return api, api.to_geodataframe(products) if output == 'pandas': return api, api.to_dataframe(products) if output == 'geojson': return api, api.to_geojson(products)
def download_tci(username, password, area_geom, outdir, **query_kw): api = sentinelsat.SentinelAPI(user=username, password=password) session = api.session logger.info('Querying SciHub') results = query.query_s2(api, area_geom, **query_kw) logger.info('Found %d products', len(results)) with concurrent.futures.ThreadPoolExecutor(max_workers=2) as exe: results_iter = tqdm.tqdm(results.values(), desc='Retrieving TCI URLs', unit='result') urls = exe.map(functools.partial(find.get_tci_url, session=session), results_iter) urls = [url for url in urls if url is not None] logger.info('Retrieved %d TCI download URLs', len(urls)) url_iter = tqdm.tqdm(urls, desc='Downloading TCI files', unit='file') targets = exe.map( functools.partial(download.download_file, outdir=outdir, session=session), url_iter) logger.info('Downloaded %d files', len(targets)) return targets
def __init__(self, datahub, catalog=None, url=None): """ :param datahub: Data source (<enum 'Datahub'>). :param catalog: Only applicable if datahub is 'STAC_local'. Can be one of the following types: Path to STAC Catalog file catalog.json (String, Path). Pystac Catalog or Collection object (pystac.catalog.Catalog, pystac.collection.Collection). None initializes an empty catalog. (default: None) :param url: Only applicable if datahub is 'STAC_API'. STAC Server endpoint, reads from STAC_API_URL environment variable by default (default: None) """ self.src = datahub if self.src == Datahub.STAC_local: # connect to STAC Catalog if isinstance( catalog, (pystac.catalog.Catalog, pystac.collection.Collection)): self.api = catalog elif isinstance(catalog, (str, Path)): href = Path(catalog).resolve().as_uri() self.api = pystac.catalog.Catalog.from_file(href) elif catalog is None: self.api = self._init_catalog() else: raise AttributeError( f"{catalog} is not a valid STAC Catalog [catalog.json, pystac.catalog.Catalog, " f"pystac.collection.Collection, None] ") elif self.src == Datahub.STAC_API: if url: self.api = StacApi(url=url) else: self.api = StacApi() elif self.src == Datahub.EarthExplorer: import landsatxplore.api # connect to Earthexplorer self.user = env_get("EARTHEXPLORER_USER") self.pw = env_get("EARTHEXPLORER_PW") self.api = landsatxplore.api.API(self.user, self.pw) elif self.src == Datahub.Scihub: # connect to Scihub self.user = env_get("SCIHUB_USER") self.pw = env_get("SCIHUB_PW") self.api = sentinelsat.SentinelAPI( self.user, self.pw, "https://apihub.copernicus.eu/apihub", show_progressbars=False, ) else: raise NotImplementedError( f"{datahub} is not supported [STAC_local, STAC_API, EarthExplorer, Scihub]" )
def get_parsed_granule_meta(auth, tile_name): api = sentinelsat.SentinelAPI(*auth) results = api.query(date=('NOW-5DAYS', 'NOW'), tileid=tile_name, platformname="Sentinel-2", producttype="S2MSI1C") r = next(iter(results.values())) metas = get_sentinel_metas(r, auth=auth) gm = s2meta.parse_granule_metadata(metadatastr=metas['granule_meta'][0]) return gm
def getlogin(): print "\nThis script requires a Copernicus Open Access Hub user ID and password." print "Please enter your credentials here.\n" print "If you don't have a login, you can register at https://scihub.copernicus.eu/\n" print "\nWarning: Some python interpreters will echo password input or fail after username input." print "To avoid this, run api_download.py from Command Prompt or Powershell.\n" coah_user = raw_input("User Name:") coah_pass = getpass.getpass("Password:") login = sentinelsat.SentinelAPI(coah_user, coah_pass, 'https://scihub.copernicus.eu/dhus') return login
def connectToAPI(username, password): ''' Connect to the SciHub API with sentinelsat. Sign up at https://scihub.copernicus.eu/. Args: username: Scihub username. password: Scihub password. ''' # Connect to Sentinel API return sentinelsat.SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
def download_from_sentinel(product, tmpdir): api = sentinelsat.SentinelAPI(user=os.environ["SENTINEL_USERNAME"], password=os.environ["SENTINEL_PASSWORD"]) sentinel_product_dir = product['filename'] if not os.path.exists(sentinel_product_dir): d_meta = api.download(product['uuid'], directory_path=tmpdir) # Extract and remove zip file zip_ref = zipfile.ZipFile(d_meta['path']) zip_ref.extractall(tmpdir) zip_ref.close() os.remove(d_meta['path']) return sentinel_product_dir
def get_sentinel_metadata_from_area(from_date, to_date, geo_json_area, cloudcoverpercentage=(0, 15)): api = sentinelsat.SentinelAPI(user=os.environ["SENTINEL_USERNAME"], password=os.environ["SENTINEL_PASSWORD"]) footprint = sentinelsat.geojson_to_wkt(geo_json_area) products = api.query(footprint, date=(from_date, to_date), platformname='Sentinel-2', producttype=('S2MS2Ap', 'S2MSI1C'), cloudcoverpercentage=cloudcoverpercentage) return products
def connectToAPI(username, password): ''' Connect to the SciHub API with sentinelsat. Sign up at https://scihub.copernicus.eu/. Args: username: Scihub username. password: Scihub password. ''' import sentinelsat # Let API be accessed by other functions global scihub_api # Connect to Sentinel API scihub_api = sentinelsat.SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
def connectToAPI(username, password): ''' Connect to the SciHub API with sentinelsat. Args: username: Scihub username. Sign up at https://scihub.copernicus.eu/. password: Scihub password. ''' # Let API be accessed by other functions global scihub_api # Disconnect from any previous session scihub_api = None # Connect to Sentinel API scihub_api = sentinelsat.SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus')
def __init__(self, datahub, datadir=None, datadir_substr=None): """ :param datahub: Data source (<enum 'Datahub'>). :param datadir: Path to directory that holds the metadata if datahub is 'File' (String). :param datadir_substr: Optional substring patterns to identify metadata in datadir if datahub is 'File' (List of String). """ self.src = datahub if self.src == Datahub.File: if not datadir: raise AttributeError( f"'datadir' has to be set if datahub is 'File'.") else: self.api = datadir if datadir_substr is None: self.api_substr = [""] else: self.api_substr = datadir_substr elif self.src == Datahub.EarthExplorer: # connect to Earthexplorer self.user = env_get("EARTHEXPLORER_USER") self.pw = env_get("EARTHEXPLORER_PW") self.api = landsatxplore.api.API(self.user, self.pw) elif self.src == Datahub.Scihub: # connect to Scihub self.user = env_get("SCIHUB_USER") self.pw = env_get("SCIHUB_PW") self.api = sentinelsat.SentinelAPI( self.user, self.pw, "https://scihub.copernicus.eu/dhus", show_progressbars=False, ) else: raise NotImplementedError( f"{datahub} is not supported [File, EarthExplorer, Scihub]")
def download_products(tiles, start_date, end_date, output_folder, show_progressbars=True): """ Descarga todos los productos del satélite Sentinel-2 para los tipos de producto S2MS2Ap y S2MSI1C :param tiles: Tiles para filtrar la descarga :param start_date: Fecha inicial en que se tomaron las imágenes :param end_date: Fecha final en que se tomaron las imágenes :param output_folder: Directorio en el que se almacenarán las imágenes :param show_progressbars: Indica si se muestran las barras de progreso durante la descarga """ print('Downloading products') api = sentinelsat.SentinelAPI(user=SENT_USER, password=SENT_PASS, api_url='https://scihub.copernicus.eu/dhus', show_progressbars=show_progressbars) query_kwargs = { 'platformname': 'Sentinel-2', 'producttype': ('S2MS2Ap', 'S2MSI1C'), 'cloudcoverpercentage': (0, 15), 'date': (start_date, end_date) } products = collections.OrderedDict() for tile in tiles: kw = query_kwargs.copy() kw['tileid'] = tile pp = api.query(**kw) products.update(pp) api.download_all(products, output_folder)
def download_products(tiles, start_date, end_date, output_folder, show_progressbars=True): """ Download all Sentinel-2 satellite products for product types S2MS2Ap and S2MSI1C :param tiles: Tiles to filter the download :param start_date: Starting date images were taken :param end_date: Final date images were taken :param output_folder: Directory in which the images will be stored :param show_progressbars: Indicates if progress bars are displayed during download """ print('Downloading products') api = sentinelsat.SentinelAPI(user=SENT_USER, password=SENT_PASS, api_url='https://scihub.copernicus.eu/dhus', show_progressbars=show_progressbars) query_kwargs = { 'platformname': 'Sentinel-2', 'producttype': ('S2MS2Ap', 'S2MSI1C'), 'cloudcoverpercentage': (0, 15), 'date': (start_date, end_date) } products = collections.OrderedDict() for tile in tiles: kw = query_kwargs.copy() kw['tileid'] = tile pp = api.query(**kw) products.update(pp) api.download_all(products, output_folder)
def get_geojson_info(products): api = sentinelsat.SentinelAPI(user=os.environ["SENTINEL_USERNAME"], password=os.environ["SENTINEL_PASSWORD"]) return api.to_geojson(products)
def get_remote_files_eumetsat(**kwargs): ''' Download swath files from EUMETSAT and store them at defined location. This fct uses the SentinelAPI for queries. ''' product = kwargs.get('product') sdate = kwargs.get('sdate') edate = kwargs.get('edate') mission = kwargs.get('mission','s3a') path_local = kwargs.get('path_local') dict_for_sub = kwargs.get('dict_for_sub') api_url = kwargs.get('api_url') import sentinelsat as ss products = None dates = (sdate.strftime('%Y-%m-%dT%H:%M:%SZ'),\ edate.strftime('%Y-%m-%dT%H:%M:%SZ')) filesort = False if path_local is None: # create local path path_template = satellite_dict[product]['dst']\ ['path_template'] strsublst = satellite_dict[product]['dst']\ ['strsub'] subdict = make_subdict(strsublst, class_object_dict=dict_for_sub) path_local = make_pathtofile(path_template,\ strsublst, subdict,\ date=sdate) filesort = True query_dict = make_query_dict(product,mission) print(query_dict) if api_url is None: api_url_lst = \ satellite_dict[product]['src']['api_url'] for url in api_url_lst: print('Source:',url) try: user, pw = get_credentials(remoteHostName=url) api = ss.SentinelAPI(user, pw, url) products = api.query(area=None, date=dates,**query_dict) break except Exception as e: print(e) else: user, pw = get_credentials(remoteHostName = api_url) api = ss.SentinelAPI(user, pw, api_url) products = api.query(area=None, date=dates,**query_dict) if products is not None: # check if download path exists if not create if not os.path.exists(path_local): os.makedirs(path_local,exist_ok=True) api.download_all(products,directory_path=path_local) #api.download(product_id) else: print('No products found!') if filesort is True: # sort files print("Data is being sorted into subdirectories " \ + "year and month ...") filelst = [f for f in os.listdir(path_local) if os.path.isfile(os.path.join(path_local,f))] sort_files(path_local,filelst,product,mission) print ('Files downloaded to: \n', path_local)
""" Script to download the Sentinel-2 images from the long time archive, for the extent of the national park """ import sentinelsat from datetime import date import logging import tkinter # login to scihub api = sentinelsat.SentinelAPI('johannes0592', 'QthBa9z7x!') nationalparc = r"F:\Studium_Trier\Masterarbeit\Datensaetze\Geojson\NationalparkGrenzenVereinfacht.json" #specify boundaries to select images footprint = sentinelsat.geojson_to_wkt(sentinelsat.read_geojson(nationalparc)) products = api.query(footprint, date=(date(2019, 10, 1), date(2020, 9, 23)), platformname='Sentinel-2', cloudcoverpercentage=(0, 60), processinglevel='Level-1C', tileid='32ULA') ids = [] x = products.keys() for item in x: ids.append(item) # start loop every 30 minutes again to order new images def downloadLTA():
# Create ring ring = ogr.Geometry(ogr.wkbLinearRing) ring.AddPoint(minX, minY) ring.AddPoint(maxX, minY) ring.AddPoint(maxX, maxY) ring.AddPoint(minX, maxY) ring.AddPoint(minX, minY) # Create polygon poly_envelope = ogr.Geometry(ogr.wkbPolygon) poly_envelope.AddGeometry(ring) return poly_envelope # Login info to the sentinel2 repository api = sentinelsat.SentinelAPI('runaas', 'FineBilder', 'https://scihub.copernicus.eu/dhus') # Login to the database with psycopg2.connect( "host=beistet port=5433 dbname=LDir user=postgres password=1234" ) as conn: with conn.cursor("geo_cur") as cur: # Do reqest # This should be narrowed down, currently it retrieves the whole database... cur.execute("""SELECT ST_AsBinary(geog), dp_fra, dp_til FROM pa""") dp_fra = dt.date.today() dp_til = dt.date.min geo_collection = ogr.Geometry(ogr.wkbMultiPolygon) for row in cur: geo_frm_db = ogr.CreateGeometryFromWkb(row[0]) if row[1] and row[1] < dp_fra:
import sentinelsat import collections # https://sentinelsat.readthedocs.io/en/stable/api.html api = sentinelsat.SentinelAPI(user="******", password="******", api_url='https://scihub.copernicus.eu/dhus', show_progressbars=True) # Reading all the tiles tiles = [] tiletxt = open("SpainPen_tiles.txt", mode="r") tilelines = tiletxt.read().splitlines() tiletxt.close() for t in tilelines: tiles.append(t) # tiles = ["29SPB"] #specify start and end date (here we use May 2017) dateStart = "20180101" dateEnd = "20181231" query_kwargs = { 'platformname': 'Sentinel-2', 'producttype': ('S2MS2Ap', 'S2MSI1C'), 'cloudcoverpercentage': (0, 15), 'date': (dateStart, dateEnd) }
import sentinelsat from datetime import date import pandas as pd import time ### Set your credentials finhub_user = '******' finhub_pwd = 'a' finhub_url = 'https://finhub.nsdc.fmi.fi' scihub_user = '******' scihub_pwd = 'a' scihub_url = 'https://scihub.copernicus.eu/dhus' ### Open API connection finhub_api = sentinelsat.SentinelAPI(finhub_user, finhub_pwd, finhub_url) scihub_api = sentinelsat.SentinelAPI(scihub_user, scihub_pwd, scihub_url) ### Search by polygon (WGS84), time, and query keywords footprint = sentinelsat.geojson_to_wkt( sentinelsat.read_geojson(r'helsinki.geojson')) startDate = date(2020, 1, 1) endDate = date(2020, 7, 30) cloudcoverage = (0, 20) platformname = 'Sentinel-2' producttype = 'S2MSI1C' # producttype='S2MSI2A' #for L2 images area_relation = "Contains" # footprint has to be fully inside the image. Other options "Intersects", "IsWithin" ### If your area is between two UTM zones, this script often downloads two versions of the same image ### Uncomment and add e.g "T35" to only focus on one UTM zone