def test_missing_dependency_dataframe(monkeypatch): api = SentinelAPI("mock_user", "mock_password") with pytest.raises(ImportError): monkeypatch.setitem(sys.modules, "pandas", None) api.to_dataframe({"test":"test"}) with pytest.raises(ImportError): monkeypatch.setitem(sys.modules, "geopandas", None) api.to_geodataframe({"test":"tst"})
def test_missing_dependency_dataframe(monkeypatch): api = SentinelAPI("mock_user", "mock_password") with pytest.raises(ImportError): monkeypatch.setitem(sys.modules, "pandas", None) api.to_dataframe({"test": "test"}) with pytest.raises(ImportError): monkeypatch.setitem(sys.modules, "geopandas", None) api.to_geodataframe({"test": "tst"})
def sentinel_query_task(parameters, task_id=None): def validate_name(string: str): return string.replace('_', '-') def order_by(order_by='ingestiondate', ascending=False): prefix_order_by = '+' if ascending else '-' order_by = prefix_order_by + order_by return order_by task = DownloadTask.objects.get(pk=task_id) platformname = parameters['products'] longitude_min, longitude_max = parameters['longitude'] latitude_min, latitude_max = parameters['latitude'] extent = Polygon([[(longitude_min, latitude_max), (longitude_max, latitude_max), (longitude_max, latitude_min), (longitude_min, latitude_min), (longitude_min, latitude_max)]]) extent = geojson_to_wkt(extent) order_by = order_by('ingestiondate', ascending=False) api = SentinelAPI(user, password, api_url) products = api.query(extent, date=(parameters['time']), platformname='SENTINEL-1', producttype='GRD', orbitdirection='DESCENDING') products_df = api.to_dataframe(products) task.execution_start = datetime.now() task.update_status('WAIT', "Download Sentinel Query. ") return products_df
def job(): #Checking Sentinel data on SciHub, based on search requirements api = SentinelAPI(oah_user, oah_pass, 'https://apihub.copernicus.eu/apihub/') count = api.count(area=wkt, date=(start_date, end_date), platformname=platformname, area_relation='Contains', raw=None, cloudcoverpercentage=(min_cloud, max_cloud), limit=20, processinglevel=processinglevel) now = datetime.now() now = now.strftime("%d/%m/%Y %H:%M:%S") print(now + ' - Checking for new data') if count > 0: # Write available image data to dataframe products = api.query(area=wkt, date=(start_date, end_date), platformname=platformname, area_relation='Contains', raw=None, cloudcoverpercentage=(min_cloud, max_cloud), limit=20, processinglevel=processinglevel) products_df = api.to_dataframe(products) detail = products_df.iat[0, 4] # Get and format important data of satellite imagery img_sat = products_df.iloc[0, 36] # Get satellite name img_proc_lvl = products_df.iloc[0, 37] # Get image processing level img_date = products_df.iloc[0, 4][6:16] # Get acquisition date img_time = products_df.iloc[0, 4][17:25] # Get acquisition time img_cloud = str(products_df.iloc[0, 21])[:5] + ' %' # Get cloud coverage #Prepare e-mail content subject = "New satellite image available - " + img_date body = "Properites if the new satellite imagery is the following.\n\n" + 'Satellite: ' + img_sat + '\n' + 'Processing level: ' + img_proc_lvl + '\n' + 'Timestamp of imagery: ' + img_date + ', ' + img_time + '\n' + 'Cloud cover percentage: ' + img_cloud message = f'Subject:{subject}\n\n{body}' #Send e-mail and go to sleep context = ssl.create_default_context() with smtplib.SMTP_SSL(smtp_server, port, context=context) as server: server.login(sender_email, password) server.sendmail(sender_email, receiver_email, message.encode("utf8")) now = datetime.now() now = now.strftime("%d/%m/%Y %H:%M:%S") print(now + ' - Mail has been sent') time.sleep(82800) #23 hours return else: # If no new image available, print message print(now + ' - There is no new data available') return
def download_sentinel(current_date, past_date, json, ndvi_tiles, df_predios, ndvi_dir, rgb_dir, ngb_dir, sentinel_dir): # Variables datos geojson_file = 'Tiles-chile.geojson' cloud_directory = '' #Storage access client = storage.Client.from_service_account_json( 'ADL-forestal-segmentation-7dc429779824.json') bucket = client.get_bucket('ranger-app') # Sentinel API api = SentinelAPI('matias-arauco', 'arauco2019', 'https://scihub.copernicus.eu/apihub/') footprint = geojson_to_wkt(json) products = api.query(footprint, date=(past_date, current_date), platformname='Sentinel-2', cloudcoverpercentage=(0, 30)) products_df = api.to_dataframe(products) products_df_sorted = products_df.loc[products_df['tileid'].isnull()] products_df_sorted = products_df_sorted.sort_values( ['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True]) print(products_df_sorted) index = products_df_sorted.index ite = 0 for i in index: file_name = products_df_sorted['title'][ite][:] year, month, day = products_df_sorted['title'][ite][ 11:15], products_df_sorted['title'][ite][ 15:17], products_df_sorted['title'][ite][17:19] tile = products_df_sorted['title'][ite][39:44] date = year + '-' + month + '-' + day print('Descargando el Tile: ', tile, ', con fecha: ', date) api.download(i, directory_path=sentinel_dir) upload_file(bucket, sentinel_dir, file_name + '.zip') ndvi_clipping(sentinel_dir, file_name, tile, date, ndvi_tiles, df_predios, ndvi_dir, rgb_dir, ngb_dir, bucket) os.remove(sentinel_dir + file_name + '.zip') ite = ite + 1
def ndvihesaplama(request): api = SentinelAPI('flavves', 'BATUhan123.', 'https://scihub.copernicus.eu/dhus') footprint = geojson_to_wkt(read_geojson('media/map.geojson')) products = api.query(footprint, date=('20191219', date(2019, 12, 29)), platformname='Sentinel-2') # pandas dataframe yap products_df = api.to_dataframe(products) # filtreleme products_df_sorted = products_df.sort_values( ['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True]) products_df_sorted = products_df_sorted.head(1) df = products_df_sorted NotDefteriKaydi = df.values.tolist() str_denemesi = str(NotDefteriKaydi) Kaydetmeye_basla = list(str_denemesi.split(",")) yerler = [0, 7, 8, 9, 12, 14, 18, 19, 20] isimler = [ "Dosya adı:", "Uydu adı", "Dosya boyutu", "Çekim tarihi", "Orbit numarası", "Bulut", "vejetasyon", "su", "not vejetasyon" ] i = 0 with open("media/books/txt/deneme.txt", "w") as dosya: for sira in yerler: print(isimler[i] + ":" + Kaydetmeye_basla[sira]) yaz = (isimler[i] + ":" + Kaydetmeye_basla[sira]) i = i + 1 dosya.write(yaz) dosya.write("\n") dosya.close() file_path = ('media\\books\\txt\\deneme.txt') #full path to text. data_file = open(file_path, 'r') data = data_file.read() data = list(data.split("\n")) context = {'deneme': data} return render(request, "todo_app/ndvihesaplama.html", context, backend().badana())
def job(): #Sentinel műholdképek keresése SciHub-on, keresési kritériumok alapján api = SentinelAPI(oah_user,oah_pass, 'https://apihub.copernicus.eu/apihub/') count=api.count(area=wkt, date=(start_date, end_date), platformname=platformname,area_relation='Contains',raw=None,cloudcoverpercentage=(min_cloud,max_cloud),limit=20, processinglevel = processinglevel) now = datetime.now() now = now.strftime("%d/%m/%Y %H:%M:%S") print(now+' - Új adat keresése') if count>0: # Elérhető műholdképek adatainak dataframe-be írása products = api.query(area=wkt, date=(start_date, end_date), platformname=platformname,area_relation='Contains',raw=None,cloudcoverpercentage=(min_cloud,max_cloud),limit=20, processinglevel = processinglevel) products_df = api.to_dataframe(products) detail=products_df.iat[0,4] # E-mailbe írandó adatok formázása img_sat=products_df.iloc[0,36] # Műhold név img_proc_lvl=products_df.iloc[0,37] # Feldolgozási szint img_date=products_df.iloc[0,4][6:16] # Felvétel rögzítésének dátuma img_time=products_df.iloc[0,4][17:25] # Felvétel rögzítésének időpontja img_cloud=str(products_df.iloc[0,21])[:5]+' %' # Felvétel felhőzöttsége #E-mail tartalom előkészítése subject="Új műholdkép - "+img_date body="A vizsgált területről készült új műholdkép adatai.\n\n"+'Műhold: '+img_sat+'\n'+'Feldolgozási szint: '+img_proc_lvl+'\n'+'Felvétel rögzítve: '+img_date+', '+img_time+'\n'+'Felvétel felhőzöttsége: '+img_cloud message=f'Subject:{subject}\n\n{body}' #E-mail küldése és keresés szüneteltetése context = ssl.create_default_context() with smtplib.SMTP_SSL(smtp_server, port, context=context) as server: server.login(sender_email, password) server.sendmail(sender_email, receiver_email, message.encode("utf8")) now = datetime.now() now = now.strftime("%d/%m/%Y %H:%M:%S") print(now+' - E-mail elküldve') time.sleep(82800) #23 óra return else: # Jelezzen ha nincs új műholdkép print(now+' - Nem érhető el új műholdkép') return
def _downloader(self, datelist, sensor, continent, obs_folder, obs_lst, uid, psw): local_obs_date = self._available_dec(obs_lst) api = SentinelAPI(uid, psw) if continent == 'Africa': footprint = geojson_to_wkt( read_geojson(r'L:/HSL/poi/AOI_Africa.geojson')) elif continent == 'West_Asia': footprint = geojson_to_wkt( read_geojson(r'L:/HSL/poi/AOI_West_Asia.geojson')) elif continent == 'Europe': footprint = geojson_to_wkt( read_geojson(r'L:/HSL/poi/AOI_Europe.geojson')) products = api.query(footprint, filename=f'S3{sensor}_*', producttype='SY_2_V10___', date=(datelist[0], datelist[-1])) products_df = api.to_dataframe(products) if products_df.size != 0: products_df_sorted = products_df.sort_values(['ingestiondate'], ascending=[True]) products_df_sorted = products_df_sorted.head(24) download_list = products_df_sorted[ ~products_df_sorted['beginposition']. isin(local_obs_date)].drop_duplicates(subset=['beginposition'], keep='last') if download_list.size != 0: downloaded = api.download_all(download_list.index, directory_path=obs_folder, n_concurrent_dl=2) if len(downloaded[2]) != 0: print('Info ! Some dataset were not downloaded') self._unzipper(obs_folder) else: pass
def satquery(geojson, date_from=None, date_to=None, platform='Sentinel-2', cloud_cover_percentage=95): """ Query products with given properties. :param geojson: str The geojson file path for footprint. :param date_from: datetime, optional :param date_to: datetime, optional The time interval filter based on the Sensing Date of the products :param platform: string 'Sentinel-1' or 'Sentinel-2' :param cloud_cover_percentage, Maximum cloud coverage percentage. Hundred percent cloud cover means no clear sky is visible. Default is 95% :return: Pandas DataFrame, Return the products from a query response as a Pandas DataFrame with the values in their appropriate Python types. """ api = SentinelAPI(USERNAME, PASSWORD, 'https://scihub.copernicus.eu/dhus') footprint = geojson_to_wkt(read_geojson(geojson), decimals=6) kwargs = dict() kwargs['platformname'] = platform if platform == 'Sentinel-1': # Level-1 Ground Range Detected (GRD) products kwargs['producttype'] = 'GRD' elif platform == 'Sentinel-2': kwargs['cloudcoverpercentage'] = (0, cloud_cover_percentage) products = api.query(footprint, date=(date_from, date_to), area_relation='Contains', **kwargs) df = api.to_dataframe(products) return df.sort_values(by='beginposition')
class Downloader: def __init__(self, str_username, str_password, str_link): self.api = SentinelAPI(str_username, str_password, str_link) self.products = None def search_polygon(self, footprint: object, str_date_start: str, str_date_end: str, str_platform_name: str, percentage: object): print('searching') self.products = self.api.query(footprint, date=(str_date_start, str_date_end), platformname=str_platform_name, cloudcoverpercentage=(percentage[0], percentage[1])) size = self.api.get_products_size(self.products) print(f'found {size}GiB of data') # print(self.products) def download_zip(self, path): self.api.download_all(self.products, path, max_attempt, True) def download_products(self, path, download_file): if download_file: self.download_zip(path) print('downloaded') df_products = self.api.to_dataframe(self.products) return df_products def download_geoproduct(self, path, download_file): if download_file: self.download_zip(path) # print('download Geos') gdf_products = self.api.to_geodataframe(self.products) return gdf_products def download_json(self): return self.api.to_geojson(self.products) def download_one(self, key, path): self.api.download(key, path, True)
def main(csvpath, apipath, qp): # Read password file # ================== try: with open(apipath) as f: (usrnam, psswrd) = f.readline().split(" ") if psswrd.endswith("\n"): psswrd = psswrd[:-1] except IOError: sys.exit("Error reading the password file!") # Connect to API using <username> and <password> # =============================================== print("Connecting to SciHub API...") api = SentinelAPI(usrnam, psswrd, "https://scihub.copernicus.eu/dhus") # Search by SciHub query keywords # =============================== products = api.query(qp['footprint'], beginposition=(qp['strtime'], qp['endtime']), endposition=(qp['strtime'], qp['endtime']), platformname=qp['platformname'], producttype=qp['producttype']) # Convert to Pandas DataFrame and sort by date ascending # ====================================================== products_df = api.to_dataframe(products) products_df_sorted = products_df.sort_values('beginposition', ascending=True) # Save to CSV file # ================ print(f"Saving list to {os.path.basename(csvpath)}") prep_csv = products_df_sorted[['uuid', 'title']] prep_csv.insert(2, "downloaded", False, allow_duplicates=True) prep_csv.to_csv(csvpath, index=False) print('Finished!')
def download_best(_box: box, download_path: str, user: str, pw: str) -> tp.List[str]: _api = SentinelAPI(user, pw, 'https://scihub.copernicus.eu/dhus') file_path = os.path.join(download_path, "save.csv") if not os.path.exists(file_path): products = _api.query( _box, date=('NOW-1MONTH', 'NOW'), platformname='Sentinel-2', processinglevel='Level-1C', cloudcoverpercentage=(0, 10), ) products_df = _api.to_dataframe(products) tile_ids = [] def _unknown_tile_id(x: str, t_ids: tp.List) -> bool: ret_val = x in t_ids if not ret_val: t_ids.append(x) return not ret_val # sort products products_df_sorted = products_df.sort_values(["cloudcoverpercentage"], ascending=[True]) # sort out tiles double tiles with higher cloud coverage first_tiles = [ _unknown_tile_id(x, tile_ids) for x in list(products_df_sorted['tileid'].array) ] # first_titles = np.vectorize(_unknown_tile_id(lambda x:x, tile_ids))(products_df_sorted['tileid'].array) products_df_sorted_unique = products_df_sorted[first_tiles] if not os.path.exists(download_path): os.makedirs(download_path) products_df_sorted_unique.to_csv(file_path) else: products_df_sorted_unique = pd.read_pickle(file_path) products_df_sorted_unique['area'] = [ __estimate_area(loads(e)) for e in list(products_df_sorted_unique['footprint'].array) ] # sort out areas smaller than three quarter of the full size of 100 km * 100 km products_df_sorted_unique_larger = products_df_sorted_unique[ products_df_sorted_unique['area'] > 100000 * 100000 / 4 * 3] _api.download_all(products_df_sorted_unique_larger.uuid, download_path) # estimate area from footprint return [ os.path.join(download_path, x) for x in products_df_sorted_unique.title ]
def test_to_pandas(products): df = SentinelAPI.to_dataframe(products) assert type(df).__name__ == "DataFrame" assert len(products) == len(df) assert set(products) == set(df.index)
class SentinelLoader: def __init__(self, dataPath, user, password, apiUrl='https://scihub.copernicus.eu/apihub/', showProgressbars=True, dateToleranceDays=5, cloudCoverage=(0,80), deriveResolutions=True, cacheApiCalls=True, cacheTilesData=True, loglevel=logging.INFO): logging.basicConfig(level=loglevel) self.api = SentinelAPI(user, password, apiUrl, show_progressbars=showProgressbars) self.dataPath = dataPath self.user = user self.password = password self.dateToleranceDays=dateToleranceDays self.cloudCoverage=cloudCoverage self.deriveResolutions=deriveResolutions self.cacheApiCalls=cacheApiCalls self.cacheTilesData=cacheTilesData def getProductBandTiles(self, geoPolygon, bandName, resolution, dateReference): """Downloads and returns file names with Sentinel2 tiles that best fit the polygon area at the desired date reference. It will perform up/downsampling if deriveResolutions is True and the desired resolution is not available for the required band.""" logger.info("Getting contents. band=%s, resolution=%s, date=%s", bandName, resolution, dateReference) #find tiles that intercepts geoPolygon within date-tolerance and date+dateTolerance dateTolerance = timedelta(days=self.dateToleranceDays) dateObj = datetime.now() if dateReference != 'now': dateObj = datetime.strptime(dateReference, '%Y-%m-%d') dateFrom = dateObj-dateTolerance dateTo = dateObj+dateTolerance resolutionDownload = resolution if self.deriveResolutions: if resolution=='10m': if bandName in ['B01', 'B09']: resolutionDownload = '60m' elif bandName in ['B05', 'B06', 'B07', 'B11', 'B12', 'B8A', 'SCL']: resolutionDownload = '20m' elif resolution=='20m': if bandName in ['B08']: resolutionDownload = '10m' elif bandName in ['B01', 'B09']: resolutionDownload = '60m' elif resolution=='60m': if bandName in ['B08']: resolutionDownload = '10m' logger.info("Querying API for candidate tiles") area = Polygon(geoPolygon).wkt #query cache key area_hash = hashlib.md5(area.encode()).hexdigest() apicache_file = self.dataPath + "/apiquery/Sentinel-2-S2MSI2A-%s-%s-%s-%s-%s.csv" % (area_hash, dateFrom.strftime("%Y%m%d"), dateTo.strftime("%Y%m%d"), self.cloudCoverage[0], self.cloudCoverage[1]) products_df = None if self.cacheApiCalls: if os.path.isfile(apicache_file): logger.debug("Using cached API query contents") products_df = pd.read_csv(apicache_file) os.system("touch -c %s" % apicache_file) else: logger.debug("Querying remote API") products = self.api.query(area, date=(dateFrom.strftime("%Y%m%d"), dateTo.strftime("%Y%m%d")), platformname='Sentinel-2', producttype='S2MSI2A', cloudcoverpercentage=self.cloudCoverage) products_df = self.api.to_dataframe(products) logger.debug("Caching API query results for later usage") saveFile(apicache_file, products_df.to_csv(index=True)) logger.debug("Found %d products", len(products_df)) if len(products_df)==0: raise Exception('Could not find any tiles for the specified parameters') products_df_sorted = products_df.sort_values(['ingestiondate','cloudcoverpercentage'], ascending=[False, False]) #select the best product. if geoPolygon() spans multiple tiles, select the best of them missing = Polygon(geoPolygon) desiredRegion = Polygon(geoPolygon) selectedTiles = [] footprints = [desiredRegion] for index, pf in products_df_sorted.iterrows(): #osgeo.ogr.Geometry footprint = gmlToPolygon(pf['gmlfootprint']) if missing.area>0: if missing.intersects(footprint)==True: missing = (missing.symmetric_difference(footprint)).difference(footprint) selectedTiles.append(index) footprints.append(footprint) if missing.area>0: raise Exception('Could not find tiles for the whole selected area at date range') logger.debug("Tiles selected for covering the entire desired area: %s", selectedTiles) # g = gpd.GeoSeries(footprints) # g.plot(cmap=plt.get_cmap('jet'), alpha=0.5) #download tiles data tileFiles = [] for index, sp in products_df.loc[selectedTiles].iterrows(): url = "https://scihub.copernicus.eu/dhus/odata/v1/Products('%s')/Nodes('%s.SAFE')/Nodes('MTD_MSIL2A.xml')/$value" % (sp['uuid'], sp['title']) meta_cache_file = self.dataPath + "/products/%s-MTD_MSIL2A.xml" % (sp['uuid']) mcontents = '' if self.cacheTilesData: if os.path.isfile(meta_cache_file): logger.debug('Reusing cached metadata info for tile \'%s\'', sp['uuid']) mcontents = loadFile(meta_cache_file) os.system("touch -c %s" % meta_cache_file) else: logger.debug('Getting metadata info for tile \'%s\' remotelly', sp['uuid']) r = requests.get(url, auth=(self.user, self.password)) if r.status_code!=200: raise Exception("Could not get metadata info. status=%s" % r.status_code) mcontents = r.content.decode("utf-8") saveFile(meta_cache_file, mcontents) rexp = "<IMAGE_FILE>GRANULE\/([0-9A-Z_]+)\/IMG_DATA\/R%s\/([0-9A-Z_]+_%s_%s)<\/IMAGE_FILE>" % (resolutionDownload, bandName, resolutionDownload) # print(mcontents) m = re.search(rexp, mcontents) if m==None: raise Exception("Could not find image metadata. uuid=%s, resolution=%s, band=%s" % (sp['uuid'], resolutionDownload, bandName)) rexp1 = "<PRODUCT_START_TIME>([\-0-9]+)T[0-9\:\.]+Z<\/PRODUCT_START_TIME>" m1 = re.search(rexp1, mcontents) if m1==None: raise Exception("Could not find product date from metadata") downloadFilename = self.dataPath + "/products/%s/%s/%s.tiff" % (m1.group(1), sp['uuid'], m.group(2)) if not os.path.exists(os.path.dirname(downloadFilename)): os.makedirs(os.path.dirname(downloadFilename)) if not self.cacheTilesData or not os.path.isfile(downloadFilename): tmp_tile_filejp2 = "%s/tmp/%s.jp2" % (self.dataPath, uuid.uuid4().hex) tmp_tile_filetiff = "%s/tmp/%s.tiff" % (self.dataPath, uuid.uuid4().hex) if not os.path.exists(os.path.dirname(tmp_tile_filejp2)): os.makedirs(os.path.dirname(tmp_tile_filejp2)) url = "https://scihub.copernicus.eu/dhus/odata/v1/Products('%s')/Nodes('%s.SAFE')/Nodes('GRANULE')/Nodes('%s')/Nodes('IMG_DATA')/Nodes('R%s')/Nodes('%s.jp2')/$value" % (sp['uuid'], sp['title'], m.group(1), resolutionDownload, m.group(2)) logger.info('Downloading tile uuid=\'%s\', resolution=\'%s\', band=\'%s\'', sp['uuid'], resolutionDownload, bandName) downloadFile(url, tmp_tile_filejp2, self.user, self.password) #remove near black features on image border due to compression artifacts. if not removed, some black pixels #will be present on final image, specially when there is an inclined crop in source images logger.debug('Removing near black compression artifacts') os.system("nearblack -o %s %s" % (tmp_tile_filetiff, tmp_tile_filejp2)) os.remove(tmp_tile_filejp2) os.system("gdal_translate %s %s" % (tmp_tile_filetiff, downloadFilename)) os.remove(tmp_tile_filetiff) else: logger.debug('Reusing tile data from cache') os.system("touch -c %s" % downloadFilename) filename = downloadFilename if resolution!=resolutionDownload: filename = self.dataPath + "/products/%s/%s/%s-%s.tiff" % (m1.group(1), sp['uuid'], m.group(2), resolution) logger.info("Resampling band %s originally in resolution %s to %s" % (bandName, resolutionDownload, resolution)) rexp = "([0-9]+).*" rnumber = re.search(rexp, resolution) if not self.cacheTilesData or not os.path.isfile(filename): os.system("gdalwarp -tr %s %s %s %s" % (rnumber.group(1), rnumber.group(1), downloadFilename, filename)) tileFiles.append(filename) return tileFiles def cropRegion(self, geoPolygon, sourceGeoTiffs): """Returns an image file with contents from a bunch of GeoTiff files cropped to the specified geoPolygon. Pay attention to the fact that a new file is created at each request and you should delete it after using it""" logger.debug("Cropping polygon %s from %d files" % (geoPolygon, len(sourceGeoTiffs))) desiredRegion = Polygon(geoPolygon) #show tile images # for fn in tilesData: # ds = gdal.Open(fn).ReadAsArray() # plt.figure(figsize=(10,10)) # plt.imshow(ds[0]) source_tiles = ' '.join(sourceGeoTiffs) tmp_file = "%s/tmp/%s.tiff" % (self.dataPath, uuid.uuid4().hex) if not os.path.exists(os.path.dirname(tmp_file)): os.makedirs(os.path.dirname(tmp_file)) #define output bounds in destination srs reference bounds = desiredRegion.bounds s1 = convertWGS84To3857(bounds[0], bounds[1]) s2 = convertWGS84To3857(bounds[2], bounds[3]) logger.debug('Combining tiles into a single image. tmpfile=%s' % tmp_file) os.system("gdalwarp -multi -srcnodata 0 -t_srs EPSG:3857 -te %s %s %s %s %s %s" % (s1[0],s1[1],s2[0],s2[1],source_tiles,tmp_file)) return tmp_file def getRegionHistory(self, geoPolygon, bandOrIndexName, resolution, dateFrom, dateTo, daysStep=5): """Gets a series of GeoTIFF files for a region for a specific band and resolution in a date range""" dateFromObj = datetime.strptime(dateFrom, '%Y-%m-%d') dateToObj = datetime.strptime(dateTo, '%Y-%m-%d') dateRef = dateFromObj regionHistoryFiles = [] while dateRef <= dateToObj: logger.debug(dateRef) dateRefStr = dateRef.strftime("%Y-%m-%d") regionFile = None if bandOrIndexName in ['NDVI', 'NDWI']: regionFile = self.getRegionIndex(geoPolygon, bandOrIndexName, resolution, dateRefStr) else: regionFile = self.getRegionBand(geoPolygon, bandOrIndexName, resolution, dateRefStr) tmp_tile_file = "%s/tmp/%s-%s-%s-%s.tiff" % (self.dataPath, dateRefStr, bandOrIndexName, resolution, uuid.uuid4().hex) os.system("mv %s %s" % (regionFile,tmp_tile_file)) regionHistoryFiles.append(tmp_tile_file) dateRef = dateRef + timedelta(days=daysStep) return regionHistoryFiles def getRegionBand(self, geoPolygon, bandName, resolution, dateReference): regionTileFiles = self.getProductBandTiles(geoPolygon, bandName, resolution, dateReference) return self.cropRegion(geoPolygon, regionTileFiles) def _getBandDataFloat(self, geoPolygon, bandName, resolution, dateReference): bandFile = self.getRegionBand(geoPolygon, bandName, resolution, dateReference) gdalBand = gdal.Open(bandFile) geoTransform = gdalBand.GetGeoTransform() projection = gdalBand.GetProjection() data = gdalBand.ReadAsArray().astype(np.float) os.remove(bandFile) return data, geoTransform, projection def getRegionIndex(self, geoPolygon, indexName, resolution, dateReference): if indexName=='NDVI': #get band 04 red,geoTransform,projection = self._getBandDataFloat(geoPolygon, 'B04', resolution, dateReference) #get band 08 nir,_,_ = self._getBandDataFloat(geoPolygon, 'B08', resolution, dateReference) #calculate ndvi ndvi = ((nir - red)/(nir + red)) #save file tmp_file = "%s/tmp/ndvi-%s.tiff" % (self.dataPath, uuid.uuid4().hex) saveGeoTiff(ndvi, tmp_file, geoTransform, projection) return tmp_file elif indexName=='NDWI': #get band 03 b03,geoTransform,projection = self._getBandDataFloat(geoPolygon, 'B03', resolution, dateReference) #get band 08 b08,_,_ = self._getBandDataFloat(geoPolygon, 'B08', resolution, dateReference) #calculate ndwi = ((b03 - b08)/(b03 + b08)) #save file tmp_file = "%s/tmp/ndwi-%s.tiff" % (self.dataPath, uuid.uuid4().hex) saveGeoTiff(ndwi, tmp_file, geoTransform, projection) return tmp_file else: raise Exception('\'indexName\' must be NDVI or NDWI') def cleanupCache(self, filesNotUsedDays): os.system("find %s -type f -name '*' -mtime +%s -exec rm {} \;" % (self.dataPath, filesNotUsedDays))
print("Dia: " + str(it.date()) + " já obtido previamente. Skipping.") continue successful = False while not successful: for tries in range(0, 5, 1): try: print("Dia: " + str(it.date())) api = SentinelAPI('amneves', 'Amnandre12') footprint = geojson_to_wkt(read_geojson('geo.geojson')) products = api.query(footprint, date=(it.date().strftime("%Y%m%d"), (it + timedelta(days=1)).date().strftime("%Y%m%d")), platformname='Sentinel-2', producttype='S2MSI1C', area_relation='Contains', cloudcoverpercentage=(0, 30)) dataframe = api.to_dataframe(products) count = dataframe.shape[0] print(str(count) + " produto(s) neste dia.") #api.download_all(products) #download(api, products) if count == 1: nome = dataframe.get_values()[0][0] p = multiprocessing.Process(target=foo, name="Foo", args=(api,products)) p.start() print("A aguardar download de " + str(nome) + ".zip") p.join(60 * 60)#60*60 = 1h if p.is_alive(): print("Demorou tempo demais. Voltar a tentar...") # Terminate foo p.terminate() p.join()
def download_sentinel1( footprint='C:\\Users\\krist\\Documents\\Syntese\\code\\notebooks\\ljundal.geojson', folder='C:\\Users\\krist\\Documents\\Syntese\\data\\s1\\download', username='******', password='******', date_start=['01', '06', '2020'], date_end=['14', '06', '2020']): ''' The following function downloads sentinel-1 data if it happens to be online. If it is online, it innitiates the LTR. The function needs a footprint. This foorprint can be made directly using e.g. modis.get_fires() function. Here, a foodprint for each fire has been made. This function also gives the date of the fire, which can be used. The function then download all Sentinel-1 images within that footprint, defined by the start and end data. The function right now donwload the data, and can initiate LRT. A later version should include automatic download after LTR. more options can be added in the api, see https://pypi.org/project/sentinelsat/. Note, from Copernicus hub, only two images can be downloaded at the same time. This is therefor run one at a time in a loop. ... Input: footprint[str]: A string with the path to a footprint in geojson format. folder[str]: The path to which the downloaded images will be saved. username[str]: Copernicus Hub username password[str]: Copernicus Hub password date_start[list]: a list with the date in string format DAY-MONTH-YEAR (with 0 if e.g. 01) date_end[list]: a list with data in strin format DAY-MONTH-YEAR (eithout 0 if e.g. 01) output: NIL: No output. images will be donwloaded. Example Author: Kristian Soerensen July 2020 [email protected] ''' #initializing API api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus') Data_start = (str(date_start[2]) + str(date_start[1]) + str(date_start[0])) #fetching all product Footprint = geojson_to_wkt(read_geojson(footprint)) #Footprint = footprint products = api.query(Footprint, date=(Data_start, date(int(date_end[2]), int(date_end[1]), int(date_end[0]))), producttype='GRD', orbitdirection='DESCENDING') cwd = os.getcwd() os.chdir(folder) #turning into dataframe products_df = api.to_dataframe(products) print(len(products_df), ' Products are found.') #checking if file in folder for i in range(len(products_df)): if os.path.isfile(products_df.iloc[i].filename) == True: #drop the file if it is in the folder products_df.drop(products_df.index[i]) print('(download_sentinel1)\n Product:', products_df.iloc[i].title, ' Already exists.') #download all products in pd, 1 at a time. for i in range(len(products_df)): api.download(products_df.iloc[i].uuid) os.chdir(cwd) return None
def sent2_query(user, passwd, geojsonfile, start_date, end_date, cloud='100', output_folder=None, api=True): """ A convenience function that wraps sentinelsat query & download Notes ----------- I have found the sentinesat sometimes fails to download the second image, so I have written some code to avoid this - choose api = False for this Parameters ----------- user : string username for esa hub passwd : string password for hub geojsonfile : string AOI polygon of interest start_date : string date of beginning of search end_date : string date of end of search output_folder : string where you intend to download the imagery cloud : string (optional) include a cloud filter in the search """ ##set up your copernicus username and password details, and copernicus download site... BE CAREFUL if you share this script with others though! api = SentinelAPI(user, passwd) # NOWT WRONG WITH API - # TODO Maybe improve check of library so it doesn't use a global # if oldsat is True: # footprint = get_coordinates(geojsonfile) # else: footprint = geojson_to_wkt(read_geojson(geojsonfile)) products = api.query(footprint, ((start_date, end_date)), platformname="Sentinel-2", cloudcoverpercentage="[0 TO " + cloud + "]") #,producttype="GRD") products_df = api.to_dataframe(products) if api is True and output_folder != None: api.download_all(directory_path=output_folder) else: prods = np.arange(len(products)) # the api was proving flaky whereas the cmd line always works hence this # is alternate the download option if output_folder != None: # procList = [] for prod in prods: #os.chdir(output_folder) sceneID = products[prod]['id'] cmd = [ 'sentinel', 'download', '-p', output_folder, user, passwd, sceneID ] print(sceneID + ' downloading') subprocess.call(cmd) #[p.wait() for p in procList] return products_df, products
def test_to_pandas(products): df = SentinelAPI.to_dataframe(products) assert '44517f66-9845-4792-a988-b5ae6e81fd3e' in df.index
def test_to_pandas(products): df = SentinelAPI.to_dataframe(products) assert type(df).__name__ == 'DataFrame' assert '44517f66-9845-4792-a988-b5ae6e81fd3e' in df.index assert len(products) == len(df)
class SentinelDownloader(object): def __init__(self, user, password, api_url='https://scihub.copernicus.eu/dhus'): try: from sentinelsat import SentinelAPI except ImportError as e: gs.fatal(_("Module requires sentinelsat library: {}").format(e)) try: import pandas except ImportError as e: gs.fatal(_("Module requires pandas library: {}").format(e)) # init logger root = logging.getLogger() root.addHandler(logging.StreamHandler(sys.stderr)) # connect SciHub via API self._api = SentinelAPI(user, password, api_url=api_url) self._products_df_sorted = None def filter(self, area, area_relation, clouds=None, producttype=None, limit=None, query={}, start=None, end=None, sortby=[], asc=True): args = {} if clouds: args['cloudcoverpercentage'] = (0, int(clouds)) if producttype: args['producttype'] = producttype if producttype.startswith('S2'): args['platformname'] = 'Sentinel-2' else: args['platformname'] = 'Sentinel-1' if not start: start = 'NOW-60DAYS' else: start = start.replace('-', '') if not end: end = 'NOW' else: end = end.replace('-', '') if query: redefined = [ value for value in args.keys() if value in query.keys() ] if redefined: gs.warning( "Query overrides already defined options ({})".format( ','.join(redefined))) args.update(query) gs.verbose( "Query: area={} area_relation={} date=({}, {}) args={}".format( area, area_relation, start, end, args)) products = self._api.query(area=area, area_relation=area_relation, date=(start, end), **args) products_df = self._api.to_dataframe(products) if len(products_df) < 1: gs.message(_('No product found')) return # sort and limit to first sorted product if sortby: self._products_df_sorted = products_df.sort_values( sortby, ascending=[asc] * len(sortby)) else: self._products_df_sorted = products_df if limit: self._products_df_sorted = self._products_df_sorted.head( int(limit)) gs.message( _('{} Sentinel product(s) found').format( len(self._products_df_sorted))) def list(self): if self._products_df_sorted is None: return for idx in range(len(self._products_df_sorted['uuid'])): if 'cloudcoverpercentage' in self._products_df_sorted: ccp = '{0:2.0f}%'.format( self._products_df_sorted['cloudcoverpercentage'][idx]) else: ccp = 'cloudcover_NA' print('{0} {1} {2} {3} {4}'.format( self._products_df_sorted['uuid'][idx], self._products_df_sorted['identifier'][idx], self._products_df_sorted['beginposition'][idx].strftime( "%Y-%m-%dT%H:%M:%SZ"), ccp, self._products_df_sorted['producttype'][idx], )) def download(self, output): if self._products_df_sorted is None: return if not os.path.exists(output): os.makedirs(output) gs.message(_('Downloading data into <{}>...').format(output)) for idx in range(len(self._products_df_sorted['uuid'])): gs.message('{} -> {}.SAFE'.format( self._products_df_sorted['uuid'][idx], os.path.join(output, self._products_df_sorted['identifier'][idx]))) # download self._api.download(self._products_df_sorted['uuid'][idx], output) def save_footprints(self, map_name): if self._products_df_sorted is None: return try: from osgeo import ogr, osr except ImportError as e: gs.fatal( _("Option <footprints> requires GDAL library: {}").format(e)) gs.message(_("Writing footprints into <{}>...").format(map_name)) driver = ogr.GetDriverByName("GPKG") tmp_name = gs.tempfile() + '.gpkg' data_source = driver.CreateDataSource(tmp_name) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) # features can be polygons or multi-polygons layer = data_source.CreateLayer(str(map_name), srs, ogr.wkbMultiPolygon) # attributes attrs = OrderedDict([("uuid", ogr.OFTString), ("ingestiondate", ogr.OFTString), ("cloudcoverpercentage", ogr.OFTInteger), ("producttype", ogr.OFTString), ("identifier", ogr.OFTString)]) for key in attrs.keys(): field = ogr.FieldDefn(key, attrs[key]) layer.CreateField(field) # features for idx in range(len(self._products_df_sorted['uuid'])): wkt = self._products_df_sorted['footprint'][idx] feature = ogr.Feature(layer.GetLayerDefn()) newgeom = ogr.CreateGeometryFromWkt(wkt) # convert polygons to multi-polygons newgeomtype = ogr.GT_Flatten(newgeom.GetGeometryType()) if newgeomtype == ogr.wkbPolygon: multigeom = ogr.Geometry(ogr.wkbMultiPolygon) multigeom.AddGeometryDirectly(newgeom) feature.SetGeometry(multigeom) else: feature.SetGeometry(newgeom) for key in attrs.keys(): if key == 'ingestiondate': value = self._products_df_sorted[key][idx].strftime( "%Y-%m-%dT%H:%M:%SZ") else: value = self._products_df_sorted[key][idx] feature.SetField(key, value) layer.CreateFeature(feature) feature = None data_source = None # coordinates of footprints are in WKT -> fp precision issues # -> snap gs.run_command('v.import', input=tmp_name, output=map_name, layer=map_name, snap=1e-10, quiet=True) def set_uuid(self, uuid_list): """Set products by uuid. TODO: Find better implementation :param uuid: uuid to download """ from sentinelsat.sentinel import SentinelAPIError self._products_df_sorted = {'uuid': []} for uuid in uuid_list: try: odata = self._api.get_product_odata(uuid, full=True) except SentinelAPIError as e: gs.error('{0}. UUID {1} skipped'.format(e, uuid)) continue for k, v in odata.items(): if k == 'id': k = 'uuid' elif k == 'Sensing start': k = 'beginposition' elif k == 'Product type': k = 'producttype' elif k == 'Cloud cover percentage': k = 'cloudcoverpercentage' elif k == 'Identifier': k = 'identifier' elif k == 'Ingestion Date': k = 'ingestiondate' elif k == 'footprint': pass else: continue if k not in self._products_df_sorted: self._products_df_sorted[k] = [] self._products_df_sorted[k].append(v)
def test_missing_dependency_dataframe(monkeypatch): with pytest.raises(ImportError): monkeypatch.setitem(sys.modules, "pandas", None) SentinelAPI.to_dataframe({"test": "test"})
class Sentinel2Loader: def __init__(self, dataPath, user, password, apiUrl='https://apihub.copernicus.eu/apihub/', showProgressbars=True, dateToleranceDays=5, cloudCoverage=(0, 80), deriveResolutions=True, cacheApiCalls=True, cacheTilesData=True, loglevel=logging.DEBUG, nirBand='B08'): logging.basicConfig(level=loglevel) self.api = SentinelAPI(user, password, apiUrl, show_progressbars=showProgressbars) self.dataPath = dataPath self.user = user self.password = password self.dateToleranceDays = dateToleranceDays self.cloudCoverage = cloudCoverage self.deriveResolutions = deriveResolutions self.cacheApiCalls = cacheApiCalls self.cacheTilesData = cacheTilesData self.nirBand = nirBand def getProductBandTiles(self, geoPolygon, bandName, resolution, dateReference): """Downloads and returns file names with Sentinel2 tiles that best fit the polygon area at the desired date reference. It will perform up/downsampling if deriveResolutions is True and the desired resolution is not available for the required band.""" logger.debug("Getting contents. band=%s, resolution=%s, date=%s", bandName, resolution, dateReference) #find tiles that intercepts geoPolygon within date-tolerance and date+dateTolerance dateTolerance = timedelta(days=self.dateToleranceDays) dateObj = datetime.now() if dateReference != 'now': dateObj = datetime.strptime(dateReference, '%Y-%m-%d') dateFrom = dateObj - dateTolerance dateTo = dateObj dateL2A = datetime.strptime('2018-12-18', '%Y-%m-%d') productLevel = '2A' if dateObj < dateL2A: logger.debug( 'Reference date %s before 2018-12-18. Will use Level1C tiles (no atmospheric correction)' % (dateObj)) productLevel = '1C' resolutionDownload = resolution if self.deriveResolutions: if productLevel == '2A': if resolution == '10m': if bandName in ['B01', 'B09']: resolutionDownload = '60m' elif bandName in [ 'B05', 'B06', 'B07', 'B11', 'B12', 'B8A', 'SCL' ]: resolutionDownload = '20m' elif resolution == '20m': if bandName in ['B08']: resolutionDownload = '10m' elif bandName in ['B01', 'B09']: resolutionDownload = '60m' elif resolution == '60m': if bandName in ['B08']: resolutionDownload = '10m' elif productLevel == '1C': resolutionDownload = '10m' logger.debug("Querying API for candidate tiles") bbox = rasterio.features.bounds(geoPolygon) geoPolygon = [(bbox[0], bbox[3]), (bbox[0], bbox[1]), (bbox[2], bbox[1]), (bbox[2], bbox[3])] area = Polygon(geoPolygon).wkt #query cache key area_hash = hashlib.md5(area.encode()).hexdigest() apicache_file = self.dataPath + "/apiquery/Sentinel-2-S2MSI%s-%s-%s-%s-%s-%s.csv" % ( productLevel, area_hash, dateFrom.strftime("%Y%m%d"), dateTo.strftime("%Y%m%d"), self.cloudCoverage[0], self.cloudCoverage[1]) products_df = None if self.cacheApiCalls: if os.path.isfile(apicache_file): logger.debug("Using cached API query contents") products_df = pd.read_csv(apicache_file) os.system("touch -c %s" % apicache_file) else: logger.debug("Querying remote API") productType = 'S2MSI%s' % productLevel products = self.api.query( area, date=(dateFrom.strftime("%Y%m%d"), dateTo.strftime("%Y%m%d")), platformname='Sentinel-2', producttype=productType, cloudcoverpercentage=self.cloudCoverage) products_df = self.api.to_dataframe(products) logger.debug("Caching API query results for later usage") saveFile(apicache_file, products_df.to_csv(index=True)) logger.debug("Found %d products", len(products_df)) if len(products_df) == 0: raise Exception( 'Could not find any tiles for the specified parameters') products_df_sorted = products_df.sort_values( ['ingestiondate', 'cloudcoverpercentage'], ascending=[False, False]) #select the best product. if geoPolygon() spans multiple tiles, select the best of them missing = Polygon(geoPolygon) desiredRegion = Polygon(geoPolygon) selectedTiles = [] footprints = [desiredRegion] for index, pf in products_df_sorted.iterrows(): #osgeo.ogr.Geometry footprint = gmlToPolygon(pf['gmlfootprint']) if missing.area > 0: if missing.intersects(footprint) == True: missing = (missing.symmetric_difference(footprint) ).difference(footprint) selectedTiles.append(index) footprints.append(footprint) if missing.area > 0: raise Exception( 'Could not find tiles for the whole selected area at date range' ) logger.debug("Tiles selected for covering the entire desired area: %s", selectedTiles) # g = gpd.GeoSeries(footprints) # g.plot(cmap=plt.get_cmap('jet'), alpha=0.5) #download tiles data tileFiles = [] for index, sp in products_df.loc[selectedTiles].iterrows(): url = "https://apihub.copernicus.eu/apihub/odata/v1/Products('%s')/Nodes('%s.SAFE')/Nodes('MTD_MSIL%s.xml')/$value" % ( sp['uuid'], sp['title'], productLevel) meta_cache_file = self.dataPath + "/products/%s-MTD_MSIL%s.xml" % ( sp['uuid'], productLevel) mcontents = '' if self.cacheTilesData and os.path.isfile(meta_cache_file): logger.debug('Reusing cached metadata info for tile \'%s\'', sp['uuid']) mcontents = loadFile(meta_cache_file) os.system("touch -c %s" % meta_cache_file) else: logger.debug('Getting metadata info for tile \'%s\' remotelly', sp['uuid']) r = requests.get(url, auth=(self.user, self.password)) if r.status_code != 200: raise Exception("Could not get metadata info. status=%s" % r.status_code) mcontents = r.content.decode("utf-8") saveFile(meta_cache_file, mcontents) rexp = "<IMAGE_FILE>GRANULE\/([0-9A-Z_]+)\/IMG_DATA\/([0-9A-Z_]+_%s)<\/IMAGE_FILE>" % ( bandName) if productLevel == '2A': rexp = "<IMAGE_FILE>GRANULE\/([0-9A-Z_]+)\/IMG_DATA\/R%s\/([0-9A-Z_]+_%s_%s)<\/IMAGE_FILE>" % ( resolutionDownload, bandName, resolutionDownload) m = re.search(rexp, mcontents) if m == None: raise Exception( "Could not find image metadata. uuid=%s, resolution=%s, band=%s" % (sp['uuid'], resolutionDownload, bandName)) rexp1 = "<PRODUCT_START_TIME>([\-0-9]+)T[0-9\:\.]+Z<\/PRODUCT_START_TIME>" m1 = re.search(rexp1, mcontents) if m1 == None: raise Exception("Could not find product date from metadata") date1 = m1.group(1) downloadFilename = self.dataPath + "/products/%s/%s/%s.tiff" % ( date1, sp['uuid'], m.group(2)) if not os.path.exists(os.path.dirname(downloadFilename)): os.makedirs(os.path.dirname(downloadFilename)) if not self.cacheTilesData or not os.path.isfile(downloadFilename): tmp_tile_filejp2 = "%s/tmp/%s.jp2" % (self.dataPath, uuid.uuid4().hex) tmp_tile_filetiff = "%s/tmp/%s.tiff" % (self.dataPath, uuid.uuid4().hex) if not os.path.exists(os.path.dirname(tmp_tile_filejp2)): os.makedirs(os.path.dirname(tmp_tile_filejp2)) if productLevel == '2A': url = "https://apihub.copernicus.eu/apihub/odata/v1/Products('%s')/Nodes('%s.SAFE')/Nodes('GRANULE')/Nodes('%s')/Nodes('IMG_DATA')/Nodes('R%s')/Nodes('%s.jp2')/$value" % ( sp['uuid'], sp['title'], m.group(1), resolutionDownload, m.group(2)) elif productLevel == '1C': url = "https://apihub.copernicus.eu/apihub/odata/v1/Products('%s')/Nodes('%s.SAFE')/Nodes('GRANULE')/Nodes('%s')/Nodes('IMG_DATA')/Nodes('%s.jp2')/$value" % ( sp['uuid'], sp['title'], m.group(1), m.group(2)) logger.info( 'Downloading tile uuid=\'%s\', resolution=\'%s\', band=\'%s\', date=\'%s\'', sp['uuid'], resolutionDownload, bandName, date1) downloadFile(url, tmp_tile_filejp2, self.user, self.password) #remove near black features on image border due to compression artifacts. if not removed, some black pixels #will be present on final image, specially when there is an inclined crop in source images if bandName == 'TCI': logger.debug('Removing near black compression artifacts') ret = os.system("which nearblack") if ret != 0: raise Exception( "gdal nearblack utility was not found in the system. install it" ) ret = os.system("which gdal_translate") if ret != 0: raise Exception( "gdal gdal_translate utility was not found in the system. install it" ) ret = os.system("nearblack -o %s %s" % (tmp_tile_filetiff, tmp_tile_filejp2)) if ret != 0: raise Exception( "Error during 'nearblack' execution. code=%d" % ret) ret = os.system("gdal_translate %s %s" % (tmp_tile_filetiff, downloadFilename)) if ret != 0: raise Exception( "Error during 'gdal_translate' execution. code=%d" % ret) os.remove(tmp_tile_filetiff) else: os.system("gdal_translate %s %s" % (tmp_tile_filejp2, downloadFilename)) os.remove(tmp_tile_filejp2) else: logger.debug('Reusing tile data from cache') os.system("touch -c %s" % downloadFilename) filename = downloadFilename if resolution != resolutionDownload: filename = self.dataPath + "/products/%s/%s/%s-%s.tiff" % ( m1.group(1), sp['uuid'], m.group(2), resolution) logger.debug( "Resampling band %s originally in resolution %s to %s" % (bandName, resolutionDownload, resolution)) rexp = "([0-9]+).*" rnumber = re.search(rexp, resolution) if not self.cacheTilesData or not os.path.isfile(filename): os.system("gdalwarp -tr %s %s %s %s" % (rnumber.group(1), rnumber.group(1), downloadFilename, filename)) tileFiles.append(filename) return tileFiles def cropRegion(self, geoPolygon, sourceGeoTiffs): """Returns an image file with contents from a bunch of GeoTiff files cropped to the specified geoPolygon. Pay attention to the fact that a new file is created at each request and you should delete it after using it""" logger.debug("Cropping polygon from %d files" % (len(sourceGeoTiffs))) desiredRegion = Polygon(geoPolygon) # #show tile images # for fn in tilesData: # ds = gdal.Open(fn).ReadAsArray() # plt.figure(figsize=(10,10)) # plt.imshow(ds[0]) source_tiles = ' '.join(sourceGeoTiffs) tmp_file = "%s/tmp/%s.tiff" % (self.dataPath, uuid.uuid4().hex) if not os.path.exists(os.path.dirname(tmp_file)): os.makedirs(os.path.dirname(tmp_file)) #define output bounds in destination srs reference bounds = desiredRegion.bounds s1 = convertWGS84To3857(bounds[0], bounds[1]) s2 = convertWGS84To3857(bounds[2], bounds[3]) logger.debug( 'Combining tiles into a single image. sources=%s tmpfile=%s' % (source_tiles, tmp_file)) os.system( "gdalwarp -multi -srcnodata 0 -t_srs EPSG:3857 -te %s %s %s %s %s %s" % (s1[0], s1[1], s2[0], s2[1], source_tiles, tmp_file)) return tmp_file def getRegionHistory(self, geoPolygon, bandOrIndexName, resolution, dateFrom, dateTo, daysStep=5, ignoreMissing=True, minVisibleLand=0, visibleLandPolygon=None, keepVisibleWithCirrus=False, interpolateMissingDates=False): """Gets a series of GeoTIFF files for a region for a specific band and resolution in a date range""" logger.info("Getting region history for band %s from %s to %s at %s" % (bandOrIndexName, dateFrom, dateTo, resolution)) dateFromObj = datetime.strptime(dateFrom, '%Y-%m-%d') dateToObj = datetime.strptime(dateTo, '%Y-%m-%d') dateRef = dateFromObj regionHistoryFiles = [] if visibleLandPolygon is None: visibleLandPolygon = geoPolygon lastSuccessfulFile = None pendingInterpolations = 0 while dateRef <= dateToObj: logger.debug(dateRef) dateRefStr = dateRef.strftime("%Y-%m-%d") regionFile = None try: cirrus = 0 if keepVisibleWithCirrus: cirrus = 1 if minVisibleLand > 0: try: labelsFile = self.getRegionBand( visibleLandPolygon, "SCL", resolution, dateRefStr) ldata = gdal.Open(labelsFile).ReadAsArray() ldata[ldata == 1] = 0 ldata[ldata == 2] = 0 ldata[ldata == 3] = 0 ldata[ldata == 4] = 1 ldata[ldata == 5] = 1 ldata[ldata == 6] = 1 ldata[ldata == 7] = 0 ldata[ldata == 8] = 0 ldata[ldata == 9] = 0 ldata[ldata == 10] = cirrus ldata[ldata == 11] = 1 os.remove(labelsFile) s = np.shape(ldata) visibleLandRatio = np.sum(ldata) / (s[0] * s[1]) if visibleLandRatio < minVisibleLand: raise Exception( "Too few land shown in image. visible ratio=%s" % visibleLandRatio) else: logger.debug( 'Minimum visible land detected. visible ratio=%s' % visibleLandRatio) except Exception as exp: logger.warning( 'Could not filter minimum visible land using SCL band. dateRef=%s err=%s' % (dateRefStr, exp)) if bandOrIndexName in [ 'NDVI', 'NDWI', 'NDWI_MacFeeters', 'NDMI' ]: regionFile = self.getRegionIndex(geoPolygon, bandOrIndexName, resolution, dateRefStr) else: regionFile = self.getRegionBand(geoPolygon, bandOrIndexName, resolution, dateRefStr) tmp_tile_file = "%s/tmp/%s-%s-%s-%s.tiff" % ( self.dataPath, dateRefStr, bandOrIndexName, resolution, uuid.uuid4().hex) useImage = True if pendingInterpolations > 0: previousData = gdal.Open(lastSuccessfulFile).ReadAsArray() nextData = gdal.Open(regionFile).ReadAsArray() # print(np.shape(previousData)) # print(np.shape(nextData)) na = np.empty(np.shape(previousData)) # print("INT") # print(np.shape(na)) #FIXME NOT WORKING. PERFORM 2D TIME INTERPOLATION raise Exception('Interpolation not yet implemented') logger.info("Calculating %s interpolated images" % pendingInterpolations) series = pd.Series([previousData]) for i in range(0, pendingInterpolations): series.add([na]) series.add([nextData]) idata = series.interpolate() # print(np.shape(idata)) pendingInterpolations = 0 #add good image os.system("mv %s %s" % (regionFile, tmp_tile_file)) regionHistoryFiles.append(tmp_tile_file) lastSuccessfulFile = tmp_tile_file except Exception as e: if ignoreMissing: logger.info( "Couldn't get data for %s using the specified filter. err=%s" % (dateRefStr, e)) else: if interpolateMissingDates: if lastSuccessfulFile != None: pendingInterpolations = pendingInterpolations + 1 else: raise e dateRef = dateRef + timedelta(days=daysStep) return regionHistoryFiles def getRegionBand(self, geoPolygon, bandName, resolution, dateReference): regionTileFiles = self.getProductBandTiles(geoPolygon, bandName, resolution, dateReference) return self.cropRegion(geoPolygon, regionTileFiles) def _getBandDataFloat(self, geoPolygon, bandName, resolution, dateReference): bandFile = self.getRegionBand(geoPolygon, bandName, resolution, dateReference) gdalBand = gdal.Open(bandFile) geoTransform = gdalBand.GetGeoTransform() projection = gdalBand.GetProjection() data = gdalBand.ReadAsArray().astype(np.float) os.remove(bandFile) return data, geoTransform, projection def getRegionIndex(self, geoPolygon, indexName, resolution, dateReference): if indexName == 'NDVI': #get band 04 red, geoTransform, projection = self._getBandDataFloat( geoPolygon, 'B04', resolution, dateReference) #get band 08 nir, _, _ = self._getBandDataFloat(geoPolygon, self.nirBand, resolution, dateReference) #calculate ndvi ndvi = ((nir - red) / (nir + red)) #save file tmp_file = "%s/tmp/ndvi-%s.tiff" % (self.dataPath, uuid.uuid4().hex) saveGeoTiff(ndvi, tmp_file, geoTransform, projection) return tmp_file elif indexName == 'NDWI': #get band 08 b08, geoTransform, projection = self._getBandDataFloat( geoPolygon, self.nirBand, resolution, dateReference) #get band 11 b11, _, _ = self._getBandDataFloat(geoPolygon, 'B11', resolution, dateReference) #calculate ndwi = ((b08 - b11) / (b08 + b11)) #save file tmp_file = "%s/tmp/ndwi-%s.tiff" % (self.dataPath, uuid.uuid4().hex) saveGeoTiff(ndwi, tmp_file, geoTransform, projection) return tmp_file elif indexName == 'NDWI_MacFeeters': #get band 03 b03, geoTransform, projection = self._getBandDataFloat( geoPolygon, 'B03', resolution, dateReference) #get band 08 b08, _, _ = self._getBandDataFloat(geoPolygon, self.nirBand, resolution, dateReference) #calculate ndwi = ((b03 - b08) / (b03 + b08)) #save file tmp_file = "%s/tmp/ndwi-%s.tiff" % (self.dataPath, uuid.uuid4().hex) saveGeoTiff(ndwi, tmp_file, geoTransform, projection) return tmp_file elif indexName == 'NDMI': #get band 03 nir, geoTransform, projection = self._getBandDataFloat( geoPolygon, 'B03', resolution, dateReference) #get band 08 swir, _, _ = self._getBandDataFloat(geoPolygon, 'B10', resolution, dateReference) #calculate ndmi = ((nir - swir) / (nir + swir)) #save file tmp_file = "%s/tmp/ndmi-%s.tiff" % (self.dataPath, uuid.uuid4().hex) saveGeoTiff(ndmi, tmp_file, geoTransform, projection) return tmp_file elif indexName == 'EVI': #https://github.com/sentinel-hub/custom-scripts/tree/master/sentinel-2 # index = 2.5 * (B08 - B04) / ((B08 + 6.0 * B04 - 7.5 * B02) + 1.0) #get band 04 b04, geoTransform, projection = self._getBandDataFloat( geoPolygon, 'B04', resolution, dateReference) #get band 08 b08, _, _ = self._getBandDataFloat(geoPolygon, self.nirBand, resolution, dateReference) #get band 02 b02, _, _ = self._getBandDataFloat(geoPolygon, 'B02', resolution, dateReference) #calculate evi = 2.5 * (b08 - b04) / ((b08 + (6.0 * b04) - (7.5 * b02)) + 1.0) #save file tmp_file = "%s/tmp/ndmi-%s.tiff" % (self.dataPath, uuid.uuid4().hex) saveGeoTiff(evi, tmp_file, geoTransform, projection) return tmp_file else: raise Exception( '\'indexName\' must be NDVI, NDWI, NDWI_MacFeeters, or NDMI') def cleanupCache(self, filesNotUsedDays): os.system("find %s -type f -name '*' -mtime +%s -exec rm {} \;" % (self.dataPath, filesNotUsedDays))
def badana(self): global kappa kappa=1 self.deneme="badana içine girdiğini gösteren kod" print(self.deneme) print(self.adana) api = SentinelAPI('flavves', 'şifre', 'https://scihub.copernicus.eu/dhus') footprint = geojson_to_wkt(read_geojson('media/map.geojson')) print(footprint) self.products = api.query(footprint,date=('20191219', date(2019, 12, 29)),platformname='Sentinel-2') products_df = api.to_dataframe(self.products) print("oluyor galiba") self.products_df_sorted = products_df.sort_values(['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True]) self.products_df_sorted = self.products_df_sorted.head(1) self.df=self.products_df_sorted self.NotDefteriKaydi = self.df.values.tolist() self.str_denemesi=str(self.NotDefteriKaydi) self.Kaydetmeye_basla=list(self.str_denemesi.split(",")) self.yerler=[0,7,8,9,12,14,18,19,20] self.isimler=["Dosya adı:","Uydu adı","Dosya boyutu","Çekim tarihi","Orbit numarası","Bulut yüzdesi","vejetasyon yüzdesi","su yüzdesi","not vejetasyon yüzdesi"] self.i=0 with open("media/books/txt/deneme.txt", "w") as self.dosya: for self.sira in self.yerler: print(self.isimler[self.i]+":"+self.Kaydetmeye_basla[self.sira]) self.yaz=(self.isimler[self.i]+":"+self.Kaydetmeye_basla[self.sira]) self.i=self.i+1 self.dosya.write(self.yaz) self.dosya.write("\n") self.dosya.close() print(self.products_df_sorted) print("indirme başladı") #burasını blockladım çünkü 1 gb arşiv indiriyor #api.download_all(self.products_df_sorted.index) print("indirme bitti") self.veri_cekme=self.products_df_sorted.index self.veri_cekme1=self.veri_cekme[0] """ Bu işlem arşivden çıkarmak için gerekli arşivin adı indirdiğimiz verinin title adı oluyor """ self.arsiv_adi=api.get_product_odata(self.veri_cekme1) self.arsiv_adi=self.arsiv_adi["title"] self.arsiv_adi=str(self.arsiv_adi) print(self.arsiv_adi) self.a = Archive(self.arsiv_adi+'.zip') self.a.extract() self.img_data_klasor_ismi=os.listdir((self.arsiv_adi+".SAFE"+'/GRANULE')) self.img_data_klasor_ismi=self.img_data_klasor_ismi[0] self.img_data_klasor_ismi=str(self.img_data_klasor_ismi) self.dosya_yer_=(self.arsiv_adi+".SAFE"+'/GRANULE/'+self.img_data_klasor_ismi+'/IMG_DATA') self.resim_isim=os.listdir(self.dosya_yer_) print(self.dosya_yer_) """ şimdi ise resimleri rasterio ile bi kullanalım """ if self.resim_isim == "R10m" or "R20m" or "R60m": self.dosya_yer_=(self.arsiv_adi+".SAFE"+'/GRANULE/'+self.img_data_klasor_ismi+'/IMG_DATA/R60m') self.resim_isim=os.listdir(self.dosya_yer_) self.resim_isim[2] self.resim_isim[3] self.jp2ler = [self.resim_isim[2],self.resim_isim[3]] self.bands = [] #burası bizim jp2 dosyalarımızı okuyacak for self.jp2 in self.jp2ler: with rasterio.open(self.dosya_yer_+"/"+self.jp2) as self.f: self.bands.append(self.f.read(1)) #resimlerimizi ayrıştırdık özel bantlara self.band_red=self.bands[0] self.band_nir=self.bands[1] print("bant değerleri hesaplandı") print(self.bands[0],self.bands[1]) else: self.resim_isim[2] self.resim_isim[3] self.jp2ler = [self.resim_isim[2],self.resim_isim[3]] self.bands = [] #burası bizim jp2 dosyalarımızı okuyacak for self.jp2 in self.jp2ler: with rasterio.open(self.dosya_yer_+"/"+self.jp2) as f: self.bands.append(self.f.read(1)) #resimlerimizi ayrıştırdık özel bantlara self.band_red=self.bands[0] self.band_nir=self.bands[1] print("bant değerleri hesaplandı") print(self.bands[0],self.bands[1]) # Klasik NDVI denklemi ile hesaplama print("ndvı hesaplanıyor") np.seterr(divide='ignore', invalid='ignore') # Calculate NDVI. This is the equation at the top of this guide expressed in code self.ndvi = (self.band_nir.astype(float) - self.band_red.astype(float)) / (self.band_nir + self.band_red) #su için yapıyorum bu analizi ## ### self.ndvi=(self.band_red.astype(float) - self.band_nir.astype(float)) / (self.band_red + self.band_nir) ### ### np.nanmin(self.ndvi), np.nanmax(self.ndvi) print("ndvı değerler aralıkları") print(np.nanmin(self.ndvi), np.nanmax(self.ndvi)) # görüntümüze bakalım renklerine ayrılmış bir görüntümüz var # çizim yapacağız bunun için gerekli kütüphaneler ekleniyor # NDVI bilindiği üzere 1 ve -1 arasındaki değerlerde sınıflandırılır. # Biz de bu değerleri renklerle göstermek istiyoruz. # Bunun için alınan sayısal değerleri farklı renk spektrumlarına atayarak elimizde NDVI için renklendirilmiş bir görüntümüz olacaktır # # Bir orta nokta belirledik ve bu sola ve sağa olacak şekilde renklendiriyoru renk spekturumunu da aşağıda paylaşacağım class RenkNormalizasyonu(colors.Normalize): def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False): self.midpoint = midpoint colors.Normalize.__init__(self, vmin, vmax, clip) def __call__(self, value, clip=None): x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1] return np.ma.masked_array(np.interp(value, x, y), np.isnan(value)) self.min=np.nanmin(self.ndvi) self.max=np.nanmax(self.ndvi) self.mid=0.1 print("bitti mi şimdi bok biter") print(self.min,self.max) self.fig = plt.figure(figsize=(20,10)) self.ax = self.fig.add_subplot(111) self.cmap = plt.cm.RdYlGn self.cax = self.ax.imshow(self.ndvi, cmap=self.cmap, clim=(self.min, self.max), norm=RenkNormalizasyonu(midpoint=self.mid,vmin=self.min, vmax=self.max)) self.ax.axis('off') self.ax.set_title('NDVI görüntüsü', fontsize=18, fontweight='bold') self.cbar = self.fig.colorbar(self.cax, orientation='horizontal', shrink=0.65) #normalde byu alttaki gibi kaydetsin ama şimdilik benim yazdığım gibi yapsın olur mu cnm muck #self.fig_kaydet="resimler/"+self.resim_isim[2]+".tif" self.fig_kaydet="media/books/covers/denemeresmi.png" self.fig.savefig(self.fig_kaydet, dpi=200, bbox_inches='tight', pad_inches=0.7) self.fig_kaydet_tif="media/books/covers/denemeresmi.tif" self.fig.savefig(self.fig_kaydet_tif, dpi=200, bbox_inches='tight', pad_inches=0.7)
class Downloader(object): def __init__(self, username, password, satellite, order_id, directory=Path('/data/')): # The connection to ESA scihub self.api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus', timeout=500.00) # Sentinel-5p currently has its own pre-operations hub self.api_s5p = SentinelAPI(user='******', password='******', api_url='https://s5phub.copernicus.eu/dhus') # Use the current datetime to name the download order self.order_id = order_id # Use ordered dict to store the metadata of the queries products self.products = OrderedDict() self.satellite = satellite self.directory = directory # if not self.directory.exists(): # Create directory if it does not exist # os.makedirs(self.directory) def query(self, footprint, startdate, enddate): if self.satellite == 's1' or self.satellite == 'all': self.query_s1(footprint, startdate, enddate) if self.satellite == 's2' or self.satellite == 'all': self.query_s2(footprint, startdate, enddate) if self.satellite == 's3' or self.satellite == 'all': self.query_s3(footprint, startdate, enddate) if self.satellite == 's5p' or self.satellite == 'all': self.query_s5p(footprint, startdate, enddate) def query_s1(self, footprint, startdate, enddate): # Define producttypes (here it is Sentinel-1 GRDH products) producttypes = ['GRD'] # Loop over producttypes and update the query dictionary # TODO: Fix this inefficient way of querying the relative orbits print(str(footprint)) if FLAGS.s2_intersection: for producttype in producttypes: queried_products = self.api.query( area=footprint, date=(startdate, enddate), platformname='Sentinel-1', #area_relation='Contains', producttype=producttype, sensoroperationalmode='IW', polarisationmode='VV VH') self.products.update(queried_products) self.intersect_products() elif FLAGS.s1_relative_orbit == [0]: for producttype in producttypes: queried_products = self.api.query( area=footprint, date=(startdate, enddate), platformname='Sentinel-1', #area_relation='Contains', producttype=producttype, sensoroperationalmode='IW', polarisationmode='VV VH') self.products.update(queried_products) else: for producttype in producttypes: for relative_orbit in FLAGS.s1_relative_orbit: queried_products = self.api.query( area=footprint, date=(startdate, enddate), platformname='Sentinel-1', producttype=producttype, #area_relation='Contains', sensoroperationalmode='IW', relativeorbitnumber=relative_orbit) self.products.update(queried_products) def query_s2(self, footprint, startdate, enddate): # Load parameters from FLAGS max_cloudcoverage = FLAGS.s2_max_cloudcoverage # Define producttypes (here it is Sentinel-2 L2A products) producttypes = [ 'S2MSI2Ap', 'S2MSI2A' ] # Producttype names differ depending on the year they were published # Loop over producttypes and update the query dictionary # TODO: Fix this inefficient way of querying the relative orbits if FLAGS.s2_relative_orbit == [0]: for producttype in producttypes: queried_products = self.api.query( footprint, date=(startdate, enddate), platformname='Sentinel-2', producttype=producttype, cloudcoverpercentage=(0, max_cloudcoverage), order_by='-ingestiondate') self.only_complete_tile(queried_products) self.products.update(queried_products) else: for producttype in producttypes: for relative_orbit in FLAGS.s2_relative_orbit: queried_products = self.api.query( footprint, date=(startdate, enddate), platformname='Sentinel-2', relativeorbitnumber=relative_orbit, producttype=producttype, cloudcoverpercentage=(0, max_cloudcoverage)) self.only_complete_tile(queried_products) self.products.update(queried_products) def query_s3(self, footprint, startdate, enddate): queried_products = self.api.query(footprint, date=(startdate, enddate), platformname='Sentinel-3', producttype='SL_2_LST___', productlevel='L2') self.products.update(queried_products) def query_s5p(self, footprint, startdate, enddate): kwargs = {} producttypedescriptions = [ 'Ozone', 'Sulphur Dioxide', 'Nitrogen Dioxide', 'Methane', 'Formaldehyde', 'Carbon Monoxide', 'Aerosol Index', 'Aerosol Layer Height', 'Cloud' ] # producttypedescriptions = ['Ozone'] # Loop over producttypes and update the query dictionary for producttypedescription in producttypedescriptions: queried_products = self.api_s5p.query( footprint, date=(startdate, enddate), platformname='Sentinel-5 Precursor', processinglevel='L2', producttypedescription=producttypedescription, **kwargs) # Remove any 'Suomi-NPP VIIRS Clouds' products which are returned as 'Cloud' (they shouldn't have been) # https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-5p/products-algorithms if producttypedescription == 'Cloud': temp_queried_products = queried_products.copy() for key in queried_products.keys(): if queried_products[key][ 'producttypedescription'] != 'Cloud': del temp_queried_products[key] queried_products = temp_queried_products self.products.update(queried_products) def print_num_and_size_of_products(self): logging.info('Number of products = ' + str(len(list(self.products)))) logging.info('Total size [GB] = ' + str(self.api.get_products_size(self.products))) # https://sentinelsat.readthedocs.io/en/master/api.html#lta-products # TODO: Get LTA retrieval to work properly (install of newest sentinelsat version is in dockerfile) # Retry every 30 min (+10 second buffertime) to request LTA products. @tenacity.retry(stop=tenacity.stop_after_attempt(200), wait=tenacity.wait_fixed(1810)) def download_zipfiles(self): zipfiles_directory = self.directory / 'zipfiles' if len(self.products) == 0: logging.info('Unable to find any products for the selected biome') sys.exit(0) return if not zipfiles_directory.exists( ): # Create directory if it does not exist os.makedirs(zipfiles_directory) # Get the products to be downloaded. The sample() funcitons permutes the dataframe, such that a new LTA product # is request at every retry. The optimal solution would have been to rearrange the dataframe by rotating the # index at every retry, but this is a quick and dirty way to achieve something similar. # (https://stackoverflow.com/a/34879805/12045808). products_df = self.queried_products_as_df().sample(frac=1) # NOTE: The code below is only useful while the Sentinel-5p has a different api than the others. After this has # been fixed, the code should be reduced to the following single line: # Download all zipfiles (it automatically checks if zipfiles already exist) # self.api.download_all(self.products, directory_path=zipfiles_directory) # Download all zipfiles # But for now, use the following code: non_s5p_products = products_df[ products_df['platformname'] != 'Sentinel-5 Precursor'] s5p_products = products_df[products_df['platformname'] == 'Sentinel-5 Precursor'] if len(non_s5p_products): logging.info("Downloading Sentinel-1/2/3 products") try: downloaded, triggered, failed = self.api.download_all( non_s5p_products.to_dict(into=OrderedDict, orient='index'), directory_path=zipfiles_directory) logging.info("Downloaded: " + str(downloaded)) logging.info("Triggered: " + str(triggered)) logging.info("failed: " + str(failed)) except InvalidChecksumError: logging.info("Error downloading products due to CheckSumError") except Exception: logging.info("Error downloading products due to unkown error") else: logging.info("No Sentinel-1/2/3 products found in query") if len(s5p_products): logging.info("Downloading Sentinel-5p products") self.api_s5p.download_all(s5p_products.to_dict(into=OrderedDict, orient='index'), directory_path=zipfiles_directory) else: logging.info("No Sentinel-5p products found in query") # The Sentinel-5p data has wrongly been given the filetype .zip, but it should be .nc, so make a copy with # .nc extension. A copy is made instead of renaming so sentinelsat doesn't re-download the file every time # it is run. s5p_downloaded_files = zipfiles_directory.glob('S5P*.zip') logging.debug( "Renaming downloaded Sentinel-5p files from .zip to .nc (due to bug in SentinelSat)" ) for file in s5p_downloaded_files: if not file.with_suffix('.nc').exists(): shutil.copy(str(file), str(file.with_suffix('.nc'))) def queried_products_as_geojson(self): return self.api.to_geojson(self.products) def only_complete_tile(self, products): found_one = False delete_list = [] for i in products: local_footprint = products.get(i).get('footprint') elements = local_footprint.split(',') if len(elements) == 5 and found_one == False: found_one = True continue else: delete_list.append(i) for i in delete_list: del products[i] def intersect_products(self): print('Found ' + str(len(self.products)) + ' products') S2_geojson_path = (self.directory / 'orders' / FLAGS.s2_order_id).with_suffix('.geojson') ground_geojsons = read_geojson(S2_geojson_path) products_geojsons = self.queried_products_as_geojson() ground_polygon = ground_geojsons.get('features')[0].get( 'geometry').get('coordinates') ground_polygon = geometry.Polygon(ground_polygon[0][0]) titles = [] ids = [] for item in products_geojsons.get('features'): id = item.get('properties').get('id') item = item.get('properties').get('title') item = (item[17:25] + item[48:55]) titles.append(item) ids.append([item, id]) unique = list(set(titles)) unique.sort() union_list = [] for i, element in enumerate(unique): local_polygon = Polygon() for j in range(len(titles)): if titles[j] == element: item = products_geojsons.get('features')[j] item = item.get('geometry').get('coordinates') item = geometry.Polygon(item[0][0]) item = affinity.scale(item, xfact=1.01, yfact=1.01) polygons = [item, local_polygon] local_polygons = unary_union(polygons) local_polygon = item union_list.append([local_polygons, element]) found_id = None for index, element in enumerate(union_list): wkt = element[0].wkt if ground_polygon.within(element[0]): found_id = element[1] break for i in ids: if found_id != i[0]: del self.products[i[1]] print('Reduced the products to ' + str(len(self.products)) + ' products') def queried_products_as_df(self): return self.api.to_dataframe(self.products) def save_queried_products(self): orders_directory = self.directory / 'orders' if not orders_directory.exists(): os.makedirs(orders_directory) # Save the queried products to a geojson file (e.g. to be loaded into QGIS) geojson_path = (self.directory / 'orders' / self.order_id).with_suffix('.geojson') with geojson_path.open('w') as geojson_file: geojson_data = self.api.to_geojson(self.products) geojson_file.write(str(geojson_data)) # Save the queried products as pandas df in a pkl file (preferred format when working in Python) df_path = (self.directory / 'orders' / self.order_id).with_suffix('.pkl') df = self.api.to_dataframe(self.products) df.to_pickle(df_path) def save_queried_products_location(self, path): path = Path(path) path = path.parent.absolute() path = path / 'log' # Save the queried products to a geojson file (e.g. to be loaded into QGIS) geojson_path = (path / self.order_id).with_suffix('.geojson') with geojson_path.open('w') as geojson_file: geojson_data = self.api.to_geojson(self.products) geojson_file.write(str(geojson_data)) # Save the queried products as pandas df in a pkl file (preferred format when working in Python) df_path = (path / self.order_id).with_suffix('.pkl') df = self.api.to_dataframe(self.products) df.to_pickle(df_path)
def test_to_pandas_empty(products): df = SentinelAPI.to_dataframe({}) assert type(df).__name__ == 'DataFrame' assert len(df) == 0
def test_to_pandas(products): df = SentinelAPI.to_dataframe(products) assert '44517f66-9845-4792-a988-b5ae6e81fd3e' in df.index
def test_to_pandas_empty(): df = SentinelAPI.to_dataframe({}) assert type(df).__name__ == "DataFrame" assert len(df) == 0
class Downloader(object): def __init__(self, username, password, satellite, order_id, directory=Path('/data/')): # The connection to ESA scihub self.api = SentinelAPI(username, password, 'https://scihub.copernicus.eu/dhus') # Sentinel-5p currently has its own pre-operations hub self.api_s5p = SentinelAPI(user='******', password='******', api_url='https://s5phub.copernicus.eu/dhus') # Use the current datetime to name the download order self.order_id = order_id # Use ordered dict to store the metadata of the queries products self.products = OrderedDict() self.satellite = satellite self.directory = directory # if not self.directory.exists(): # Create directory if it does not exist # os.makedirs(self.directory) def query(self, footprint, startdate, enddate): if self.satellite == 's1' or self.satellite == 'all': self.query_s1(footprint, startdate, enddate) if self.satellite == 's2' or self.satellite == 'all': self.query_s2(footprint, startdate, enddate) if self.satellite == 's3' or self.satellite == 'all': self.query_s3(footprint, startdate, enddate) if self.satellite == 's5p' or self.satellite == 'all': self.query_s5p(footprint, startdate, enddate) def query_s1(self, footprint, startdate, enddate): # Define producttypes (here it is Sentinel-1 GRDH products) producttypes = ['GRD'] # Loop over producttypes and update the query dictionary # TODO: Fix this inefficient way of querying the relative orbits if FLAGS.s1_relative_orbit == [0]: for producttype in producttypes: queried_products = self.api.query(footprint, date=(startdate, enddate), platformname='Sentinel-1', producttype=producttype, sensoroperationalmode='IW') self.products.update(queried_products) else: for producttype in producttypes: for relative_orbit in FLAGS.s1_relative_orbit: queried_products = self.api.query( footprint, date=(startdate, enddate), platformname='Sentinel-1', producttype=producttype, sensoroperationalmode='IW', relativeorbitnumber=relative_orbit) self.products.update(queried_products) def query_s2(self, footprint, startdate, enddate): # Load parameters from FLAGS max_cloudcoverage = FLAGS.s2_max_cloudcoverage # Define producttypes (here it is Sentinel-2 L2A products) producttypes = [ 'S2MSI2Ap', 'S2MSI2A' ] # Producttype names differ depending on the year they were published # Loop over producttypes and update the query dictionary # TODO: Fix this inefficient way of querying the relative orbits if FLAGS.s2_relative_orbit == [0]: for producttype in producttypes: queried_products = self.api.query( footprint, date=(startdate, enddate), platformname='Sentinel-2', producttype=producttype, cloudcoverpercentage=(0, max_cloudcoverage)) self.products.update(queried_products) else: for producttype in producttypes: for relative_orbit in FLAGS.s2_relative_orbit: queried_products = self.api.query( footprint, date=(startdate, enddate), platformname='Sentinel-2', relativeorbitnumber=relative_orbit, producttype=producttype, cloudcoverpercentage=(0, max_cloudcoverage)) self.products.update(queried_products) def query_s3(self, footprint, startdate, enddate): queried_products = self.api.query(footprint, date=(startdate, enddate), platformname='Sentinel-3', producttype='SL_2_LST___', productlevel='L2') self.products.update(queried_products) def query_s5p(self, footprint, startdate, enddate): kwargs = {} producttypedescriptions = [ 'Ozone', 'Sulphur Dioxide', 'Nitrogen Dioxide', 'Methane', 'Formaldehyde', 'Carbon Monoxide', 'Aerosol Index', 'Aerosol Layer Height', 'Cloud' ] # producttypedescriptions = ['Ozone'] # Loop over producttypes and update the query dictionary for producttypedescription in producttypedescriptions: queried_products = self.api_s5p.query( footprint, date=(startdate, enddate), platformname='Sentinel-5 Precursor', processinglevel='L2', producttypedescription=producttypedescription, **kwargs) # Remove any 'Suomi-NPP VIIRS Clouds' products which are returned as 'Cloud' (they shouldn't have been) # https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-5p/products-algorithms if producttypedescription == 'Cloud': temp_queried_products = queried_products.copy() for key in queried_products.keys(): if queried_products[key][ 'producttypedescription'] != 'Cloud': del temp_queried_products[key] queried_products = temp_queried_products self.products.update(queried_products) def print_num_and_size_of_products(self): logging.info('Number of products = ' + str(len(list(self.products)))) logging.info('Total size [GB] = ' + str(self.api.get_products_size(self.products))) # https://sentinelsat.readthedocs.io/en/master/api.html#lta-products # TODO: Get LTA retrieval to work properly (install of newest sentinelsat version is in dockerfile) # Retry every 30 min (+10 second buffertime) to request LTA products. @tenacity.retry(stop=tenacity.stop_after_attempt(200), wait=tenacity.wait_fixed(1810)) def download_zipfiles(self): zipfiles_directory = self.directory / 'zipfiles' if not zipfiles_directory.exists( ): # Create directory if it does not exist os.makedirs(zipfiles_directory) # Get the products to be downloaded. The sample() funcitons permutes the dataframe, such that a new LTA product # is request at every retry. The optimal solution would have been to rearrange the dataframe by rotating the # index at every retry, but this is a quick and dirty way to achieve something similar. # (https://stackoverflow.com/a/34879805/12045808). products_df = self.queried_products_as_df().sample(frac=1) # NOTE: The code below is only useful while the Sentinel-5p has a different api than the others. After this has # been fixed, the code should be reduced to the following single line: # Download all zipfiles (it automatically checks if zipfiles already exist) # self.api.download_all(self.products, directory_path=zipfiles_directory) # Download all zipfiles # But for now, use the following code: non_s5p_products = products_df[ products_df['platformname'] != 'Sentinel-5 Precursor'] s5p_products = products_df[products_df['platformname'] == 'Sentinel-5 Precursor'] if len(non_s5p_products): logging.info("Downloading Sentinel-1/2/3 products") self.api.download_all(non_s5p_products.to_dict(into=OrderedDict, orient='index'), directory_path=zipfiles_directory) else: logging.info("No Sentinel-1/2/3 products found in query") if len(s5p_products): logging.info("Downloading Sentinel-5p products") self.api_s5p.download_all(s5p_products.to_dict(into=OrderedDict, orient='index'), directory_path=zipfiles_directory) else: logging.info("No Sentinel-5p products found in query") # The Sentinel-5p data has wrongly been given the filetype .zip, but it should be .nc, so make a copy with # .nc extension. A copy is made instead of renaming so sentinelsat doesn't re-download the file every time # it is run. s5p_downloaded_files = zipfiles_directory.glob('S5P*.zip') logging.debug( "Renaming downloaded Sentinel-5p files from .zip to .nc (due to bug in SentinelSat)" ) for file in s5p_downloaded_files: if not file.with_suffix('.nc').exists(): shutil.copy(str(file), str(file.with_suffix('.nc'))) def queried_products_as_geojson(self): return self.api.to_geojson(self.products) def queried_products_as_df(self): return self.api.to_dataframe(self.products) def save_queried_products(self): orders_directory = self.directory / 'orders' if not orders_directory.exists(): os.makedirs(orders_directory) # Save the queried products to a geojson file (e.g. to be loaded into QGIS) geojson_path = (self.directory / 'orders' / self.order_id).with_suffix('.geojson') with geojson_path.open('w') as geojson_file: geojson_data = self.api.to_geojson(self.products) geojson_file.write(str(geojson_data)) # Save the queried products as pandas df in a pkl file (preferred format when working in Python) df_path = (self.directory / 'orders' / self.order_id).with_suffix('.pkl') df = self.api.to_dataframe(self.products) df.to_pickle(df_path)
def test_to_pandas(products): df = SentinelAPI.to_dataframe(products) assert type(df).__name__ == 'DataFrame' assert len(products) == len(df) assert set(products) == set(df.index)
def test_to_pandas_empty(products): df = SentinelAPI.to_dataframe({}) assert type(df).__name__ == 'DataFrame' assert len(df) == 0
class SentinelDownloader(object): def __init__(self, user, password, api_url='https://scihub.copernicus.eu/apihub'): self._apiname = api_url self._user = user self._password = password # init logger root = logging.getLogger() root.addHandler(logging.StreamHandler( sys.stderr )) if self._apiname == 'https://scihub.copernicus.eu/apihub': try: from sentinelsat import SentinelAPI except ImportError as e: gs.fatal(_("Module requires sentinelsat library: {}").format(e)) # connect SciHub via API self._api = SentinelAPI(self._user, self._password, api_url=self._apiname ) elif self._apiname == 'USGS_EE': try: import landsatxplore.api from landsatxplore.errors import EarthExplorerError except ImportError as e: gs.fatal(_("Module requires landsatxplore library: {}").format(e)) api_login = False while api_login is False: # avoid login conflict in possible parallel execution try: self._api = landsatxplore.api.API(self._user, self._password) api_login = True except EarthExplorerError as e: time.sleep(1) self._products_df_sorted = None def filter(self, area, area_relation, clouds=None, producttype=None, limit=None, query={}, start=None, end=None, sortby=[], asc=True, relativeorbitnumber=None): args = {} if clouds: args['cloudcoverpercentage'] = (0, int(clouds)) if relativeorbitnumber: args['relativeorbitnumber'] = relativeorbitnumber if producttype.startswith('S2') and int(relativeorbitnumber) > 143: gs.warning("This relative orbit number is out of range") elif int(relativeorbitnumber) > 175: gs.warning(_("This relative orbit number is out of range")) if producttype: args['producttype'] = producttype if producttype.startswith('S2'): args['platformname'] = 'Sentinel-2' else: args['platformname'] = 'Sentinel-1' if not start: start = 'NOW-60DAYS' else: start = start.replace('-', '') if not end: end = 'NOW' else: end = end.replace('-', '') if query: redefined = [value for value in args.keys() if value in query.keys()] if redefined: gs.warning(_("Query overrides already defined options ({})").format( ','.join(redefined) )) args.update(query) gs.verbose(_("Query: area={} area_relation={} date=({}, {}) args={}").format( area, area_relation, start, end, args )) products = self._api.query( area=area, area_relation=area_relation, date=(start, end), **args ) products_df = self._api.to_dataframe(products) if len(products_df) < 1: gs.message(_("No product found")) return # sort and limit to first sorted product if sortby: self._products_df_sorted = products_df.sort_values( sortby, ascending=[asc] * len(sortby) ) else: self._products_df_sorted = products_df if limit: self._products_df_sorted = self._products_df_sorted.head(int(limit)) gs.message(_("{} Sentinel product(s) found").format(len(self._products_df_sorted))) def list(self): if self._products_df_sorted is None: return id_kw = ('uuid', 'entity_id') identifier_kw = ('identifier', 'display_id') cloud_kw = ('cloudcoverpercentage', 'cloud_cover') time_kw = ('beginposition', 'acquisition_date') kw_idx = 1 if self._apiname == 'USGS_EE' else 0 for idx in range(len(self._products_df_sorted[id_kw[kw_idx]])): if cloud_kw[kw_idx] in self._products_df_sorted: ccp = '{0:2.0f}%'.format( float(self._products_df_sorted[cloud_kw[kw_idx]][idx])) else: ccp = 'cloudcover_NA' print_str = '{0} {1}'.format( self._products_df_sorted[id_kw[kw_idx]][idx], self._products_df_sorted[identifier_kw[kw_idx]][idx]) if kw_idx == 1: time_string = self._products_df_sorted[time_kw[kw_idx]][idx] else: time_string = self._products_df_sorted[ time_kw[kw_idx]][idx].strftime("%Y-%m-%dT%H:%M:%SZ") print_str += ' {0} {1}'.format(time_string, ccp) if kw_idx == 0: print_str += ' {0}'.format( self._products_df_sorted['producttype'][idx]) print(print_str) def download(self, output, sleep=False, maxretry=False, datasource='ESA_COAH'): if self._products_df_sorted is None: return create_dir(output) gs.message(_("Downloading data into <{}>...").format(output)) if datasource == 'USGS_EE': from landsatxplore.earthexplorer import EarthExplorer from landsatxplore.errors import EarthExplorerError from zipfile import ZipFile ee_login = False while ee_login is False: # avoid login conflict in possible parallel execution try: ee = EarthExplorer(self._user, self._password) ee_login = True except EarthExplorerError as e: time.sleep(1) for idx in range(len(self._products_df_sorted['entity_id'])): scene = self._products_df_sorted['entity_id'][idx] identifier = self._products_df_sorted['display_id'][idx] zip_file = os.path.join(output, '{}.zip'.format(identifier)) gs.message(_("Downloading {}...").format(identifier)) try: ee.download(identifier=identifier, output_dir=output, timeout=600) except EarthExplorerError as e: gs.fatal(_(e)) ee.logout() # extract .zip to get "usual" .SAFE with ZipFile(zip_file, 'r') as zip: safe_name = zip.namelist()[0].split('/')[0] outpath = os.path.join(output, safe_name) zip.extractall(path=output) gs.message(_("Downloaded to <{}>").format(outpath)) try: os.remove(zip_file) except Exception as e: gs.warning(_("Unable to remove {0}:{1}").format( zip_file, e)) elif datasource == "ESA_COAH": for idx in range(len(self._products_df_sorted['uuid'])): gs.message('{} -> {}.SAFE'.format( self._products_df_sorted['uuid'][idx], os.path.join(output, self._products_df_sorted['identifier'][idx]) )) # download out = self._api.download(self._products_df_sorted['uuid'][idx], output) if sleep: x = 1 online = out['Online'] while not online: # sleep is in minutes so multiply by 60 time.sleep(int(sleep) * 60) out = self._api.download(self._products_df_sorted['uuid'][idx], output) x += 1 if x > maxretry: online = True elif datasource == 'GCS': for scene_id in self._products_df_sorted['identifier']: gs.message(_("Downloading {}...").format(scene_id)) dl_code = download_gcs(scene_id, output) if dl_code == 0: gs.message(_("Downloaded to {}").format( os.path.join(output, '{}.SAFE'.format(scene_id)))) else: # remove incomplete file del_folder = os.path.join(output, '{}.SAFE'.format(scene_id)) try: shutil.rmtree(del_folder) except Exception as e: gs.warning(_("Unable to removed unfinished " "download {}".format(del_folder))) def save_footprints(self, map_name): if self._products_df_sorted is None: return if self._apiname == 'USGS_EE': gs.fatal(_( "USGS Earth Explorer does not support footprint download.")) try: from osgeo import ogr, osr except ImportError as e: gs.fatal(_("Option <footprints> requires GDAL library: {}").format(e)) gs.message(_("Writing footprints into <{}>...").format(map_name)) driver = ogr.GetDriverByName("GPKG") tmp_name = gs.tempfile() + '.gpkg' data_source = driver.CreateDataSource(tmp_name) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) # features can be polygons or multi-polygons layer = data_source.CreateLayer(str(map_name), srs, ogr.wkbMultiPolygon) # attributes attrs = OrderedDict([ ("uuid", ogr.OFTString), ("ingestiondate", ogr.OFTString), ("cloudcoverpercentage", ogr.OFTInteger), ("producttype", ogr.OFTString), ("identifier", ogr.OFTString) ]) # Sentinel-1 data does not have cloudcoverpercentage prod_types = [type for type in self._products_df_sorted["producttype"]] s1_types = ["SLC", "GRD"] if any(type in prod_types for type in s1_types): del attrs["cloudcoverpercentage"] for key in attrs.keys(): field = ogr.FieldDefn(key, attrs[key]) layer.CreateField(field) # features for idx in range(len(self._products_df_sorted['uuid'])): wkt = self._products_df_sorted['footprint'][idx] feature = ogr.Feature(layer.GetLayerDefn()) newgeom = ogr.CreateGeometryFromWkt(wkt) # convert polygons to multi-polygons newgeomtype = ogr.GT_Flatten(newgeom.GetGeometryType()) if newgeomtype == ogr.wkbPolygon: multigeom = ogr.Geometry(ogr.wkbMultiPolygon) multigeom.AddGeometryDirectly(newgeom) feature.SetGeometry(multigeom) else: feature.SetGeometry(newgeom) for key in attrs.keys(): if key == 'ingestiondate': value = self._products_df_sorted[key][idx].strftime("%Y-%m-%dT%H:%M:%SZ") else: value = self._products_df_sorted[key][idx] feature.SetField(key, value) layer.CreateFeature(feature) feature = None data_source = None # coordinates of footprints are in WKT -> fp precision issues # -> snap gs.run_command('v.import', input=tmp_name, output=map_name, layer=map_name, snap=1e-10, quiet=True ) def get_products_from_uuid_usgs(self, uuid_list): scenes = [] for uuid in uuid_list: metadata = self._api.metadata(uuid, 'SENTINEL_2A') scenes.append(metadata) scenes_df = pandas.DataFrame.from_dict(scenes) self._products_df_sorted = scenes_df gs.message(_("{} Sentinel product(s) found").format( len(self._products_df_sorted))) def set_uuid(self, uuid_list): """Set products by uuid. TODO: Find better implementation :param uuid: uuid to download """ if self._apiname == 'USGS_EE': self.get_products_from_uuid_usgs(uuid_list) else: from sentinelsat.sentinel import SentinelAPIError self._products_df_sorted = {'uuid': []} for uuid in uuid_list: try: odata = self._api.get_product_odata(uuid, full=True) except SentinelAPIError as e: gs.error(_("{0}. UUID {1} skipped".format(e, uuid))) continue for k, v in odata.items(): if k == 'id': k = 'uuid' elif k == 'Sensing start': k = 'beginposition' elif k == 'Product type': k = 'producttype' elif k == 'Cloud cover percentage': k = 'cloudcoverpercentage' elif k == 'Identifier': k = 'identifier' elif k == 'Ingestion Date': k = 'ingestiondate' elif k == 'footprint': pass else: continue if k not in self._products_df_sorted: self._products_df_sorted[k] = [] self._products_df_sorted[k].append(v) def filter_USGS(self, area, area_relation, clouds=None, producttype=None, limit=None, query={}, start=None, end=None, sortby=[], asc=True, relativeorbitnumber=None): if area_relation != 'Intersects': gs.fatal(_( "USGS Earth Explorer only supports area_relation" " 'Intersects'")) if relativeorbitnumber: gs.fatal(_( "USGS Earth Explorer does not support 'relativeorbitnumber'" " option.")) if producttype and producttype != 'S2MSI1C': gs.fatal(_( "USGS Earth Explorer only supports producttype S2MSI1C")) if query: if not any(key in query for key in ['identifier', 'filename', 'usgs_identifier']): gs.fatal(_( "USGS Earth Explorer only supports query options" " 'filename', 'identifier' or 'usgs_identifier'.")) if 'usgs_identifier' in query: # get entityId from usgs identifier and directly save results usgs_id = query['usgs_identifier'] check_s2l1c_identifier(usgs_id, source='usgs') # entity_id = self._api.lookup('SENTINEL_2A', [usgs_id], # inverse=True) entity_id = self._api.get_entity_id([usgs_id], 'SENTINEL_2A') self.get_products_from_uuid_usgs(entity_id) return else: if "filename" in query: esa_id = query['filename'].replace('.SAFE', '') else: esa_id = query['identifier'] check_s2l1c_identifier(esa_id, source='esa') esa_prod_id = esa_id.split('_')[-1] utm_tile = esa_id.split('_')[-2] acq_date = esa_id.split('_')[2].split('T')[0] acq_date_string = '{0}-{1}-{2}'.format( acq_date[:4], acq_date[4:6], acq_date[6:]) start_date = end_date = acq_date_string # build the USGS style S2-identifier if utm_tile.startswith('T'): utm_tile_base = utm_tile[1:] bbox = get_bbox_from_S2_UTMtile(utm_tile_base) else: # get coordinate pairs from wkt string str_1 = 'POLYGON((' str_2 = '))' coords = area[area.find(str_1)+len(str_1):area.rfind(str_2)].split(',') # add one space to first pair for consistency coords[0] = ' ' + coords[0] lons = [float(pair.split(' ')[1]) for pair in coords] lats = [float(pair.split(' ')[2]) for pair in coords] bbox = (min(lons), min(lats), max(lons), max(lats)) start_date = start end_date = end usgs_args = { 'dataset': 'SENTINEL_2A', 'bbox': bbox, 'start_date': start_date, 'end_date': end_date } if clouds: usgs_args['max_cloud_cover'] = clouds if limit: usgs_args['max_results'] = limit scenes = self._api.search(**usgs_args) self._api.logout() if query: # check if the UTM-Tile is correct, remove otherwise for scene in scenes: if scene['display_id'].split('_')[1] != utm_tile: scenes.remove(scene) # remove redundant scene if len(scenes) == 2: for scene in scenes: prod_id = scene['display_id'].split('_')[-1] if prod_id != esa_prod_id: scenes.remove(scene) if len(scenes) < 1: gs.message(_("No product found")) return scenes_df = pandas.DataFrame.from_dict(scenes) if sortby: # replace sortby keywords with USGS keywords for idx, keyword in enumerate(sortby): if keyword == 'cloudcoverpercentage': sortby[idx] = 'cloud_cover' # turn cloudcover to float to make it sortable scenes_df['cloud_cover'] = pandas.to_numeric( scenes_df['cloud_cover']) elif keyword == 'ingestiondate': sortby[idx] = 'acquisition_date' # what does sorting by footprint mean elif keyword == 'footprint': sortby[idx] = 'display_id' self._products_df_sorted = scenes_df.sort_values( sortby, ascending=[asc] * len(sortby), ignore_index=True ) else: self._products_df_sorted = scenes_df gs.message(_("{} Sentinel product(s) found").format( len(self._products_df_sorted)))
class SentinelDownloader(object): def __init__( self, user, password, api_url="https://apihub.copernicus.eu/apihub", cred_req=True, ): self._apiname = api_url self._user = user self._password = password self._cred_req = cred_req # init logger root = logging.getLogger() root.addHandler(logging.StreamHandler(sys.stderr)) if self._apiname not in ["USGS_EE", "GCS"]: try: from sentinelsat import SentinelAPI except ImportError as e: gs.fatal( _("Module requires sentinelsat library: {}").format(e)) # connect SciHub via API self._api = SentinelAPI(self._user, self._password, api_url=self._apiname) elif self._apiname == "USGS_EE": api_login = False while api_login is False: # avoid login conflict in possible parallel execution try: self._api = landsatxplore.api.API(self._user, self._password) api_login = True except EarthExplorerError as e: time.sleep(1) self._products_df_sorted = None def filter( self, area, area_relation, clouds=None, producttype=None, limit=None, query={}, start=None, end=None, sortby=[], asc=True, relativeorbitnumber=None, ): # Dict to identify plaforms from requested product platforms = { "SL": "Sentinel-1", "GR": "Sentinel-1", "OC": "Sentinel-1", "S2": "Sentinel-2", "S3": "Sentinel-3", } args = {} if clouds: args["cloudcoverpercentage"] = (0, int(clouds)) if relativeorbitnumber: args["relativeorbitnumber"] = relativeorbitnumber if producttype.startswith("S2") and int(relativeorbitnumber) > 143: gs.warning("This relative orbit number is out of range") elif int(relativeorbitnumber) > 175: gs.warning(_("This relative orbit number is out of range")) if producttype: if producttype.startswith("S3"): # Using custom product names for Sentinel-3 products that look less cryptic split = [0, 2, 4, 5, 8] args["producttype"] = "_".join([ producttype[i:j] for i, j in zip(split, split[1:] + [None]) ][1:]).ljust(11, "_") else: args["producttype"] = producttype args["platformname"] = platforms[producttype[0:2]] if not start: start = "NOW-60DAYS" else: start = start.replace("-", "") if not end: end = "NOW" else: end = end.replace("-", "") if query: redefined = [ value for value in args.keys() if value in query.keys() ] if redefined: gs.warning( _("Query overrides already defined options ({})").format( ",".join(redefined))) args.update(query) gs.verbose( _("Query: area={} area_relation={} date=({}, {}) args={}").format( area, area_relation, start, end, args)) if self._cred_req is False: # in the main function it is ensured that there is an "identifier" query self._products_df_sorted = pandas.DataFrame( {"identifier": [query["identifier"]]}) return products = self._api.query(area=area, area_relation=area_relation, date=(start, end), **args) products_df = self._api.to_dataframe(products) if len(products_df) < 1: gs.message(_("No product found")) return # sort and limit to first sorted product if sortby: self._products_df_sorted = products_df.sort_values( sortby, ascending=[asc] * len(sortby)) else: self._products_df_sorted = products_df if limit: self._products_df_sorted = self._products_df_sorted.head( int(limit)) gs.message( _("{} Sentinel product(s) found").format( len(self._products_df_sorted))) def list(self): if self._products_df_sorted is None: return id_kw = ("uuid", "entity_id") identifier_kw = ("identifier", "display_id") cloud_kw = ("cloudcoverpercentage", "cloud_cover") time_kw = ("beginposition", "acquisition_date") kw_idx = 1 if self._apiname == "USGS_EE" else 0 for idx in range(len(self._products_df_sorted[id_kw[kw_idx]])): if cloud_kw[kw_idx] in self._products_df_sorted: ccp = "{0:2.0f}%".format( float(self._products_df_sorted[cloud_kw[kw_idx]][idx])) else: ccp = "cloudcover_NA" print_str = "{0} {1}".format( self._products_df_sorted[id_kw[kw_idx]][idx], self._products_df_sorted[identifier_kw[kw_idx]][idx], ) if kw_idx == 1: time_string = self._products_df_sorted[time_kw[kw_idx]][idx] else: time_string = self._products_df_sorted[ time_kw[kw_idx]][idx].strftime("%Y-%m-%dT%H:%M:%SZ") print_str += " {0} {1}".format(time_string, ccp) if kw_idx == 0: print_str += " {0}".format( self._products_df_sorted["producttype"][idx]) print_str += " {0}".format( self._products_df_sorted["size"][idx]) print(print_str) def skip_existing(self, output, pattern_file): prod_df_type = type(self._products_df_sorted) # Check i skipping is possible/required if prod_df_type != dict: if self._products_df_sorted.empty: return elif not self._products_df_sorted or os.path.exists(output) == False: return # Check if ingestion date is returned by API if "ingestiondate" not in self._products_df_sorted: gs.warning( _("Ingestiondate not returned. Cannot filter previously downloaded scenes" )) return # Check for previously downloaded scenes existing_files = [ f for f in os.listdir(output) if re.search(r".zip$|.safe$|.ZIP$|.SAFE$", f) ] if len(existing_files) <= 1: return # Filter by ingestion date skiprows = [] for idx, display_id in enumerate( self._products_df_sorted["identifier"]): existing_file = [ sfile for sfile in existing_files if display_id in sfile ] if existing_file: creation_time = datetime.fromtimestamp( os.path.getctime(existing_file[0])) if self._products_df_sorted["ingestiondate"][ idx] <= creation_time: gs.message( _("Skipping scene: {} which is already downloaded.". format(self._products_df_sorted["identifier"][idx]))) skiprows.append(display_id) if prod_df_type == dict: for scene in skiprows: idx = self._products_df_sorted["identifier"].index(scene) for key in self._products_df_sorted: self._products_df_sorted[key].pop(idx) else: self._products_df_sorted = self._products_df_sorted[ ~self._products_df_sorted["identifier"].isin(skiprows)] def download(self, output, sleep=False, maxretry=False, datasource="ESA_COAH"): if self._products_df_sorted is None: return create_dir(output) gs.message(_("Downloading data into <{}>...").format(output)) if datasource == "USGS_EE": from landsatxplore.earthexplorer import EarthExplorer from landsatxplore.errors import EarthExplorerError from zipfile import ZipFile ee_login = False while ee_login is False: # avoid login conflict in possible parallel execution try: ee = EarthExplorer(self._user, self._password) ee_login = True except EarthExplorerError as e: time.sleep(1) for idx in range(len(self._products_df_sorted["entity_id"])): scene = self._products_df_sorted["entity_id"][idx] identifier = self._products_df_sorted["display_id"][idx] zip_file = os.path.join(output, "{}.zip".format(identifier)) gs.message(_("Downloading {}...").format(identifier)) try: ee.download(identifier=identifier, output_dir=output, timeout=600) except EarthExplorerError as e: gs.fatal(_(e)) ee.logout() # extract .zip to get "usual" .SAFE with ZipFile(zip_file, "r") as zip: safe_name = zip.namelist()[0].split("/")[0] outpath = os.path.join(output, safe_name) zip.extractall(path=output) gs.message(_("Downloaded to <{}>").format(outpath)) try: os.remove(zip_file) except Exception as e: gs.warning( _("Unable to remove {0}:{1}").format(zip_file, e)) elif datasource == "ESA_COAH": for idx in range(len(self._products_df_sorted["uuid"])): gs.message("{} -> {}.SAFE".format( self._products_df_sorted["uuid"][idx], os.path.join(output, self._products_df_sorted["identifier"][idx]), )) # download out = self._api.download(self._products_df_sorted["uuid"][idx], output) if sleep: x = 1 online = out["Online"] while not online: # sleep is in minutes so multiply by 60 time.sleep(int(sleep) * 60) out = self._api.download( self._products_df_sorted["uuid"][idx], output) x += 1 if x > maxretry: online = True elif datasource == "GCS": for scene_id in self._products_df_sorted["identifier"]: gs.message(_("Downloading {}...").format(scene_id)) dl_code = download_gcs(scene_id, output) if dl_code == 0: gs.message( _("Downloaded to {}").format( os.path.join(output, "{}.SAFE".format(scene_id)))) else: # remove incomplete file del_folder = os.path.join(output, "{}.SAFE".format(scene_id)) try: shutil.rmtree(del_folder) except Exception as e: gs.warning( _("Unable to removed unfinished " "download {}".format(del_folder))) def save_footprints(self, map_name): if self._products_df_sorted is None: return if self._apiname == "USGS_EE": gs.fatal( _("USGS Earth Explorer does not support footprint download.")) try: from osgeo import ogr, osr except ImportError as e: gs.fatal( _("Option <footprints> requires GDAL library: {}").format(e)) gs.message(_("Writing footprints into <{}>...").format(map_name)) driver = ogr.GetDriverByName("GPKG") tmp_name = gs.tempfile() + ".gpkg" data_source = driver.CreateDataSource(tmp_name) srs = osr.SpatialReference() srs.ImportFromEPSG(4326) # features can be polygons or multi-polygons layer = data_source.CreateLayer(str(map_name), srs, ogr.wkbMultiPolygon) # attributes attrs = OrderedDict([ ("uuid", ogr.OFTString), ("ingestiondate", ogr.OFTString), ("cloudcoverpercentage", ogr.OFTInteger), ("producttype", ogr.OFTString), ("identifier", ogr.OFTString), ]) # Sentinel-1 data does not have cloudcoverpercentage prod_types = [type for type in self._products_df_sorted["producttype"]] if not any(type in prod_types for type in cloudcover_products): del attrs["cloudcoverpercentage"] for key in attrs.keys(): field = ogr.FieldDefn(key, attrs[key]) layer.CreateField(field) # features for idx in range(len(self._products_df_sorted["uuid"])): wkt = self._products_df_sorted["footprint"][idx] feature = ogr.Feature(layer.GetLayerDefn()) newgeom = ogr.CreateGeometryFromWkt(wkt) # convert polygons to multi-polygons newgeomtype = ogr.GT_Flatten(newgeom.GetGeometryType()) if newgeomtype == ogr.wkbPolygon: multigeom = ogr.Geometry(ogr.wkbMultiPolygon) multigeom.AddGeometryDirectly(newgeom) feature.SetGeometry(multigeom) else: feature.SetGeometry(newgeom) for key in attrs.keys(): if key == "ingestiondate": value = self._products_df_sorted[key][idx].strftime( "%Y-%m-%dT%H:%M:%SZ") else: value = self._products_df_sorted[key][idx] feature.SetField(key, value) layer.CreateFeature(feature) feature = None data_source = None # coordinates of footprints are in WKT -> fp precision issues # -> snap gs.run_command( "v.import", input=tmp_name, output=map_name, layer=map_name, snap=1e-10, quiet=True, ) def get_products_from_uuid_usgs(self, uuid_list): scenes = [] for uuid in uuid_list: metadata = self._api.metadata(uuid, "SENTINEL_2A") scenes.append(metadata) scenes_df = pandas.DataFrame.from_dict(scenes) self._products_df_sorted = scenes_df gs.message( _("{} Sentinel product(s) found").format( len(self._products_df_sorted))) def set_uuid(self, uuid_list): """Set products by uuid. TODO: Find better implementation :param uuid: uuid to download """ if self._apiname == "USGS_EE": self.get_products_from_uuid_usgs(uuid_list) else: from sentinelsat.sentinel import SentinelAPIError self._products_df_sorted = {"uuid": []} for uuid in uuid_list: try: odata = self._api.get_product_odata(uuid, full=True) except SentinelAPIError as e: gs.error(_("{0}. UUID {1} skipped".format(e, uuid))) continue for k, v in odata.items(): if k == "id": k = "uuid" elif k == "Sensing start": k = "beginposition" elif k == "Product type": k = "producttype" elif k == "Cloud cover percentage": k = "cloudcoverpercentage" elif k == "Identifier": k = "identifier" elif k == "Ingestion Date": k = "ingestiondate" elif k == "footprint": pass else: continue if k not in self._products_df_sorted: self._products_df_sorted[k] = [] self._products_df_sorted[k].append(v) def filter_USGS( self, area, area_relation, clouds=None, producttype=None, limit=None, query={}, start=None, end=None, sortby=[], asc=True, relativeorbitnumber=None, ): if area_relation != "Intersects": gs.fatal( _("USGS Earth Explorer only supports area_relation" " 'Intersects'")) if relativeorbitnumber: gs.fatal( _("USGS Earth Explorer does not support 'relativeorbitnumber'" " option.")) if producttype and producttype != "S2MSI1C": gs.fatal( _("USGS Earth Explorer only supports producttype S2MSI1C")) if query: if not any( key in query for key in ["identifier", "filename", "usgs_identifier"]): gs.fatal( _("USGS Earth Explorer only supports query options" " 'filename', 'identifier' or 'usgs_identifier'.")) if "usgs_identifier" in query: # get entityId from usgs identifier and directly save results usgs_id = query["usgs_identifier"] check_s2l1c_identifier(usgs_id, source="usgs") # entity_id = self._api.lookup('SENTINEL_2A', [usgs_id], # inverse=True) entity_id = self._api.get_entity_id([usgs_id], "SENTINEL_2A") self.get_products_from_uuid_usgs(entity_id) return else: if "filename" in query: esa_id = query["filename"].replace(".SAFE", "") else: esa_id = query["identifier"] check_s2l1c_identifier(esa_id, source="esa") esa_prod_id = esa_id.split("_")[-1] utm_tile = esa_id.split("_")[-2] acq_date = esa_id.split("_")[2].split("T")[0] acq_date_string = "{0}-{1}-{2}".format(acq_date[:4], acq_date[4:6], acq_date[6:]) start_date = end_date = acq_date_string # build the USGS style S2-identifier if utm_tile.startswith("T"): utm_tile_base = utm_tile[1:] bbox = get_bbox_from_S2_UTMtile(utm_tile_base) else: # get coordinate pairs from wkt string str_1 = "POLYGON((" str_2 = "))" coords = area[area.find(str_1) + len(str_1):area.rfind(str_2)].split(",") # add one space to first pair for consistency coords[0] = " " + coords[0] lons = [float(pair.split(" ")[1]) for pair in coords] lats = [float(pair.split(" ")[2]) for pair in coords] bbox = (min(lons), min(lats), max(lons), max(lats)) start_date = start end_date = end usgs_args = { "dataset": "SENTINEL_2A", "bbox": bbox, "start_date": start_date, "end_date": end_date, } if clouds: usgs_args["max_cloud_cover"] = clouds if limit: usgs_args["max_results"] = limit scenes = self._api.search(**usgs_args) self._api.logout() if query: # check if the UTM-Tile is correct, remove otherwise for scene in scenes: if scene["display_id"].split("_")[1] != utm_tile: scenes.remove(scene) # remove redundant scene if len(scenes) == 2: for scene in scenes: prod_id = scene["display_id"].split("_")[-1] if prod_id != esa_prod_id: scenes.remove(scene) if len(scenes) < 1: gs.message(_("No product found")) return scenes_df = pandas.DataFrame.from_dict(scenes) if sortby: # replace sortby keywords with USGS keywords for idx, keyword in enumerate(sortby): if keyword == "cloudcoverpercentage": sortby[idx] = "cloud_cover" # turn cloudcover to float to make it sortable scenes_df["cloud_cover"] = pandas.to_numeric( scenes_df["cloud_cover"]) elif keyword == "ingestiondate": sortby[idx] = "acquisition_date" # what does sorting by footprint mean elif keyword == "footprint": sortby[idx] = "display_id" self._products_df_sorted = scenes_df.sort_values(sortby, ascending=[asc] * len(sortby), ignore_index=True) else: self._products_df_sorted = scenes_df gs.message( _("{} Sentinel product(s) found").format( len(self._products_df_sorted)))
# from pysal.contrib.viz import mapping as maps # connect to the API api = SentinelAPI('phillr', 'testme2019', 'https://scihub.copernicus.eu/dhus') # search by polygon, time, and SciHub query keywords footprint = geojson_to_wkt(read_geojson('sample-polygone.geojson')) products = api.query(footprint, date=('20190101', date(2019, 2, 27)), platformname='Sentinel-2', cloudcoverpercentage=(0, 10), processinglevel='Level-1C') # todo check for processed data len(products) # convert to Pandas DataFrame products_df = api.to_dataframe(products) # sort for most recent and lowest cloud cover products_df_sorted = products_df.sort_values(['ingestiondate', 'cloudcoverpercentage'], ascending=[True, True]) test = products_df_sorted.head(1) test['cloudcoverpercentage'] test['ingestiondate'] set(products_df['processinglevel']) # GeoJSON FeatureCollection containing footprints and metadata of the scenes geojson_products = api.to_geojson(products) # GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries geodata_products = api.to_geodataframe(products)
def sent2_amazon(user, passwd, geojsonfile, start_date, end_date, output_folder, tile=None, cloud='100'): """ Query the ESA catalogue then download S2 from AWS with correct renaming of stuff Uses joblib to parallelise multiple files from aws Way quicker than ESA-based download Notes: ------------------------ Credit to sentinelsat for the query aspect of this function, and sentinelhub for the AWS aspect. Parameters ---------- user : string username for esa hub passwd : string password for hub geojsonfile : string AOI polygon of interest start_date : string date of beginning of search end_date : string date of end of search output_folder : string where you intend to download the imagery tile : string S2 tile cloud : string (optional) include a cloud filter in the search """ # Examples of sentinehub usage: #download_safe_format('S2A_OPER_PRD_MSIL1C_PDMC_20160121T043931_R069_V20160103T171947_20160103T171947') #download_safe_format('S2A_MSIL1C_20170414T003551_N0204_R016_T54HVH_20170414T003551') #download_safe_format(tile=('T38TML','2015-12-19'), entire_product=True) #entire prod really mean whole tile in old format! Avoid! #download_safe_format(tile=('T54HVH','2017-04-14')) # Use sentinel sat to query api = SentinelAPI(user, passwd) # if oldsat is True: # footprint = get_coordinates(geojsonfile) # else: footprint = geojson_to_wkt(read_geojson(geojsonfile)) products = api.query(footprint, ((start_date, end_date)), platformname="Sentinel-2", cloudcoverpercentage="[0 TO " + cloud + "]") #,producttype="GRD") products_df = api.to_dataframe(products) # If using an aoi shape this is the option to follow at present until I # write a native function if tile is None: Parallel(n_jobs=-1, verbose=2)( delayed(download_safe_format)(i, folder=output_folder) for i in products_df.identifier) # If the tile id is known then use this - likely handy for oldfmt else: # A kludge for now until I spend more than two mins writing this func dateList = [] for prod in products_df['ingestiondate']: date1 = prod.strftime('%Y-%m-%d') dateList.append(date1) Parallel(n_jobs=-1, verbose=2)( delayed(download_safe_format)(tile=(tile, i), folder=output_folder) for i in dateList) return products_df, products